diff --git a/.gitignore b/.gitignore
index 19578645a63..c1981871785 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,19 @@
 cmd/mimirtool/mimirtool
+cmd/mimirtool/mimirtool_linux_amd64
+cmd/mimirtool/mimirtool_linux_arm64
 cmd/test-exporter/test-exporter
 cmd/mimir/mimir
+cmd/mimir/mimir_linux_amd64
+cmd/mimir/mimir_linux_arm64
 cmd/query-tee/query-tee
+cmd/query-tee/query-tee_linux_amd64
+cmd/query-tee/query-tee_linux_arm64
 cmd/metaconvert/metaconvert
+cmd/metaconvert/metaconvert_linux_amd64
+cmd/metaconvert/metaconvert_linux_arm64
 cmd/mimir-continuous-test/mimir-continuous-test
+cmd/mimir-continuous-test/mimir-continuous-test_linux_amd64
+cmd/mimir-continuous-test/mimir-continuous-test_linux_arm64
 .uptodate
 .pkg
 .cache
diff --git a/Makefile b/Makefile
index 3dde46ca888..f9fe5e369db 100644
--- a/Makefile
+++ b/Makefile
@@ -97,13 +97,18 @@ SED ?= $(shell which gsed 2>/dev/null || which sed)
 	@touch $@
 
 # This target compiles mimir for linux/amd64 and linux/arm64 and then builds and pushes a multiarch image to the target repository.
-# We don't separate building of single-platform and multiplatform images here (as we do for push-multiarch-build-image), as
-# Mimir's Dockerfile is not doing much, and is unlikely to fail.
-push-multiarch-mimir:
-	@echo
-	$(MAKE) GOOS=linux GOARCH=amd64 BINARY_SUFFIX=_linux_amd64 cmd/mimir/mimir
-	$(MAKE) GOOS=linux GOARCH=arm64 BINARY_SUFFIX=_linux_arm64 cmd/mimir/mimir
-	$(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) --build-arg=USE_BINARY_SUFFIX=true -t $(IMAGE_PREFIX)mimir:$(IMAGE_TAG) cmd/mimir
+# We don't do separate building of single-platform and multiplatform images here (as we do for push-multiarch-build-image), as
+# these Dockerfiles are not doing much, and are unlikely to fail.
+push-multiarch-%/$(UPTODATE):
+	$(eval DIR := $(patsubst push-multiarch-%/$(UPTODATE),%,$@))
+
+	if [ -f $(DIR)/main.go ]; then \
+		$(MAKE) GOOS=linux GOARCH=amd64 BINARY_SUFFIX=_linux_amd64 $(DIR)/$(shell basename $(DIR)); \
+		$(MAKE) GOOS=linux GOARCH=arm64 BINARY_SUFFIX=_linux_arm64 $(DIR)/$(shell basename $(DIR)); \
+	fi
+	$(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) --build-arg=USE_BINARY_SUFFIX=true -t $(IMAGE_PREFIX)$(shell basename $(DIR)):$(IMAGE_TAG) $(DIR)/
+
+push-multiarch-mimir: push-multiarch-cmd/mimir/.uptodate
 
 # This target fetches current build image, and tags it with "latest" tag. It can be used instead of building the image locally.
 .PHONY: fetch-build-image
@@ -396,6 +401,9 @@ format-makefiles: $(MAKE_FILES)
 clean:
 	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
 	rm -rf -- $(UPTODATE_FILES) $(EXES) .cache dist
+	# Remove executables built for multiarch images.
+	find . -type f -name '*_linux_arm64' -perm +u+x -exec rm {} \;
+	find . -type f -name '*_linux_amd64' -perm +u+x -exec rm {} \;
 	go clean ./...
 
 clean-protos:
diff --git a/cmd/metaconvert/Dockerfile b/cmd/metaconvert/Dockerfile
index b02dbed0886..4acc3391f57 100644
--- a/cmd/metaconvert/Dockerfile
+++ b/cmd/metaconvert/Dockerfile
@@ -5,7 +5,13 @@
 
 FROM       alpine:3.15.0
 RUN        apk add --no-cache ca-certificates
-COPY       metaconvert /
+# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG.
+ARG        TARGETOS
+ARG        TARGETARCH
+ARG        BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}"
+# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix.
+ARG        USE_BINARY_SUFFIX
+COPY       metaconvert${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /metaconvert
 ENTRYPOINT ["/metaconvert"]
 
 ARG revision
diff --git a/cmd/mimir-continuous-test/Dockerfile b/cmd/mimir-continuous-test/Dockerfile
index f0b0b8b339a..3f26c8e951a 100644
--- a/cmd/mimir-continuous-test/Dockerfile
+++ b/cmd/mimir-continuous-test/Dockerfile
@@ -2,7 +2,13 @@
 
 FROM       alpine:3.15.0
 RUN        apk add --no-cache ca-certificates
-COPY       mimir-continuous-test /
+# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG.
+ARG        TARGETOS
+ARG        TARGETARCH
+ARG        BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}"
+# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix.
+ARG        USE_BINARY_SUFFIX
+COPY       mimir-continuous-test${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /mimir-continuous-test
 ENTRYPOINT ["/mimir-continuous-test"]
 
 ARG revision
diff --git a/cmd/mimirtool/Dockerfile b/cmd/mimirtool/Dockerfile
index 3d029782786..55a6257353a 100644
--- a/cmd/mimirtool/Dockerfile
+++ b/cmd/mimirtool/Dockerfile
@@ -6,7 +6,7 @@ RUN        apk add --no-cache ca-certificates
 ARG        TARGETOS
 ARG        TARGETARCH
 ARG        BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}"
-# Set to non-empty value to use ${TARGET_SUFFIX} when copying mimir binary, leave unset to use no suffix.
+# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix.
 ARG        USE_BINARY_SUFFIX
 COPY       mimirtool${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /bin/mimirtool
 ENTRYPOINT [ "/bin/mimirtool" ]
diff --git a/cmd/query-tee/Dockerfile b/cmd/query-tee/Dockerfile
index 8571fb2bc95..8b039303f88 100644
--- a/cmd/query-tee/Dockerfile
+++ b/cmd/query-tee/Dockerfile
@@ -5,7 +5,13 @@
 
 FROM       alpine:3.15.0
 RUN        apk add --no-cache ca-certificates
-COPY       query-tee /
+# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG.
+ARG        TARGETOS
+ARG        TARGETARCH
+ARG        BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}"
+# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix.
+ARG        USE_BINARY_SUFFIX
+COPY       query-tee${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /query-tee
 ENTRYPOINT ["/query-tee"]
 
 ARG revision