diff --git a/test/README.md b/test/README.md
index 31fb4946b862..0005ba30e23f 100644
--- a/test/README.md
+++ b/test/README.md
@@ -99,6 +99,8 @@ go test -v -tags=e2e -count=1 ./test/e2e --resolvabledomain
 
 ### Building the test images
 
+Note: this is only required when you run conformance/e2e tests locally with `go test` commands.
+
 The [`upload-test-images.sh`](./upload-test-images.sh) script can be used to build and push the
 test images used by the conformance and e2e tests. It requires:
 
@@ -110,18 +112,12 @@ test images used by the conformance and e2e tests. It requires:
 To run the script for all end to end test images:
 
 ```bash
-./test/upload-test-images.sh ./test/e2e/test_images ./test/conformance/test_images
+./test/upload-test-images.sh
 ```
 
 ### Adding new test images
 
-New test images should be placed in their own subdirectories. Be sure to to include a `Dockerfile`
-for building and running the test image.
-
-The new test images will also need to be uploaded to the e2e tests Docker repo. You will need one
-of the owners found in [`/test/OWNERS`](OWNERS) to do this.
-
-Because the test images are uploaded to the same folder, they **must** have different names.
+New test images should be placed in `./test/test_images`.
 
 ## Flags
 
diff --git a/test/conformance/test_images/pizzaplanetv1/Dockerfile b/test/conformance/test_images/pizzaplanetv1/Dockerfile
deleted file mode 100644
index 5de3f532e178..000000000000
--- a/test/conformance/test_images/pizzaplanetv1/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2018 The Knative Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM golang AS builder
-
-WORKDIR /go/src/pizzaplanetv1
-ADD . /go/src/pizzaplanetv1
-
-RUN CGO_ENABLED=0 go build .
-
-
-FROM gcr.io/distroless/base
-
-COPY --from=builder /go/src/pizzaplanetv1/pizzaplanetv1 /entrypoint
-
-ENTRYPOINT ["/entrypoint"]
diff --git a/test/conformance/test_images/pizzaplanetv2/Dockerfile b/test/conformance/test_images/pizzaplanetv2/Dockerfile
deleted file mode 100644
index f314e658226b..000000000000
--- a/test/conformance/test_images/pizzaplanetv2/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2018 The Knative Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM golang AS builder
-
-WORKDIR /go/src/pizzaplanetv2
-ADD . /go/src/pizzaplanetv2
-
-RUN CGO_ENABLED=0 go build .
-
-
-FROM gcr.io/distroless/base
-
-COPY --from=builder /go/src/pizzaplanetv2/pizzaplanetv2 /entrypoint
-
-ENTRYPOINT ["/entrypoint"]
diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh
index e16e86edf039..5ad923ebc36f 100755
--- a/test/e2e-tests.sh
+++ b/test/e2e-tests.sh
@@ -109,6 +109,13 @@ function dump_extra_cluster_state() {
   kubectl logs $(get_app_pod controller knative-serving)
 }
 
+function publish_test_images() {
+  image_dirs="$(find ${REPO_ROOT_DIR}/test/test_images -mindepth 1 -maxdepth 1 -type d)"
+  for image_dir in ${image_dirs}; do
+    ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
+  done
+}
+
 # Script entry point.
 
 initialize $@
@@ -121,6 +128,8 @@ header "Building and starting Knative Serving"
 export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
 create_everything
 
+publish_test_images
+
 # Handle test failures ourselves, so we can dump useful info.
 set +o errexit
 set +o pipefail
@@ -138,7 +147,6 @@ options=""
 report_go_test \
   -v -tags=e2e -count=1 -timeout=20m \
   ./test/conformance ./test/e2e \
-  ${options} \
-  -dockerrepo gcr.io/knative-tests/test-images/knative-serving || fail_test
+  ${options} || fail_test
 
 success
diff --git a/test/e2e/helloworld_shell_test.go b/test/e2e/helloworld_shell_test.go
index 277a336878eb..03424cb3a2aa 100644
--- a/test/e2e/helloworld_shell_test.go
+++ b/test/e2e/helloworld_shell_test.go
@@ -32,7 +32,7 @@ import (
 )
 
 const (
-	appYaml              = "test_images/helloworld/helloworld.yaml"
+	appYaml              = "../test_images/helloworld/helloworld.yaml"
 	yamlImagePlaceholder = "github.com/knative/serving/test_images/helloworld"
 	ingressTimeout       = 5 * time.Minute
 	servingTimeout       = 2 * time.Minute
diff --git a/test/e2e/test_images/README.md b/test/e2e/test_images/README.md
deleted file mode 100644
index e6df9295fa79..000000000000
--- a/test/e2e/test_images/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Test images node
-
-The subdirectories contain the test images used in the conformance tests.
-
-For details about building and adding new images, see the [section about test
-images](/test/README.md#test-images).
-
diff --git a/test/e2e/test_images/autoscale/Dockerfile b/test/e2e/test_images/autoscale/Dockerfile
deleted file mode 100644
index a0c5121ee5ff..000000000000
--- a/test/e2e/test_images/autoscale/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM golang:latest
-RUN mkdir /app
-ADD . /app/
-WORKDIR /app
-RUN go build -o autoscale .
-CMD ["/app/autoscale"]
diff --git a/test/e2e/test_images/helloworld/Dockerfile b/test/e2e/test_images/helloworld/Dockerfile
deleted file mode 100644
index 5a61955739d2..000000000000
--- a/test/e2e/test_images/helloworld/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM golang:latest
-RUN mkdir /app
-ADD . /app/
-WORKDIR /app
-RUN go build -o helloworld .
-CMD ["/app/helloworld"]
diff --git a/test/e2e/test_images/httpproxy/Dockerfile b/test/e2e/test_images/httpproxy/Dockerfile
deleted file mode 100644
index 55bb46647161..000000000000
--- a/test/e2e/test_images/httpproxy/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM golang:latest
-RUN mkdir /app
-ADD . /app/
-WORKDIR /app
-RUN go build -o httpproxy .
-CMD ["/app/httpproxy"]
diff --git a/test/e2e_flags.go b/test/e2e_flags.go
index f05c912bdbb7..021098baf047 100644
--- a/test/e2e_flags.go
+++ b/test/e2e_flags.go
@@ -46,7 +46,7 @@ func initializeFlags() *EnvironmentFlags {
 	flag.StringVar(&f.Cluster, "cluster", defaultCluster,
 		"Provide the cluster to test against. Defaults to $K8S_CLUSTER_OVERRIDE, then current cluster in kubeconfig if $K8S_CLUSTER_OVERRIDE is unset.")
 
-	defaultRepo := os.Getenv("DOCKER_REPO_OVERRIDE")
+	defaultRepo := path.Join(os.Getenv("DOCKER_REPO_OVERRIDE"), "github.com/knative/serving/test/test_images")
 	flag.StringVar(&f.DockerRepo, "dockerrepo", defaultRepo,
 		"Provide the uri of the docker repo you have uploaded the test image to using `uploadtestimage.sh`. Defaults to $DOCKER_REPO_OVERRIDE")
 
diff --git a/test/conformance/test_images/README.md b/test/test_images/README.md
similarity index 91%
rename from test/conformance/test_images/README.md
rename to test/test_images/README.md
index e6df9295fa79..81cb2a3a55a5 100644
--- a/test/conformance/test_images/README.md
+++ b/test/test_images/README.md
@@ -1,6 +1,6 @@
 # Test images node
 
-The subdirectories contain the test images used in the conformance tests.
+The subdirectories contain the test images used in the conformance and e2e tests.
 
 For details about building and adding new images, see the [section about test
 images](/test/README.md#test-images).
diff --git a/test/e2e/test_images/autoscale/README.md b/test/test_images/autoscale/README.md
similarity index 100%
rename from test/e2e/test_images/autoscale/README.md
rename to test/test_images/autoscale/README.md
diff --git a/test/e2e/test_images/autoscale/autoscale.go b/test/test_images/autoscale/autoscale.go
similarity index 100%
rename from test/e2e/test_images/autoscale/autoscale.go
rename to test/test_images/autoscale/autoscale.go
diff --git a/test/e2e/test_images/helloworld/README.md b/test/test_images/helloworld/README.md
similarity index 100%
rename from test/e2e/test_images/helloworld/README.md
rename to test/test_images/helloworld/README.md
diff --git a/test/e2e/test_images/helloworld/helloworld.go b/test/test_images/helloworld/helloworld.go
similarity index 100%
rename from test/e2e/test_images/helloworld/helloworld.go
rename to test/test_images/helloworld/helloworld.go
diff --git a/test/e2e/test_images/helloworld/helloworld.yaml b/test/test_images/helloworld/helloworld.yaml
similarity index 100%
rename from test/e2e/test_images/helloworld/helloworld.yaml
rename to test/test_images/helloworld/helloworld.yaml
diff --git a/test/e2e/test_images/httpproxy/README.md b/test/test_images/httpproxy/README.md
similarity index 100%
rename from test/e2e/test_images/httpproxy/README.md
rename to test/test_images/httpproxy/README.md
diff --git a/test/e2e/test_images/httpproxy/httpproxy.go b/test/test_images/httpproxy/httpproxy.go
similarity index 100%
rename from test/e2e/test_images/httpproxy/httpproxy.go
rename to test/test_images/httpproxy/httpproxy.go
diff --git a/test/conformance/test_images/pizzaplanetv1/README.md b/test/test_images/pizzaplanetv1/README.md
similarity index 100%
rename from test/conformance/test_images/pizzaplanetv1/README.md
rename to test/test_images/pizzaplanetv1/README.md
diff --git a/test/conformance/test_images/pizzaplanetv1/main.go b/test/test_images/pizzaplanetv1/main.go
similarity index 100%
rename from test/conformance/test_images/pizzaplanetv1/main.go
rename to test/test_images/pizzaplanetv1/main.go
diff --git a/test/conformance/test_images/pizzaplanetv2/README.md b/test/test_images/pizzaplanetv2/README.md
similarity index 100%
rename from test/conformance/test_images/pizzaplanetv2/README.md
rename to test/test_images/pizzaplanetv2/README.md
diff --git a/test/conformance/test_images/pizzaplanetv2/main.go b/test/test_images/pizzaplanetv2/main.go
similarity index 100%
rename from test/conformance/test_images/pizzaplanetv2/main.go
rename to test/test_images/pizzaplanetv2/main.go
diff --git a/test/upload-test-images.sh b/test/upload-test-images.sh
index 3be305f6abfb..9ba07410ca09 100755
--- a/test/upload-test-images.sh
+++ b/test/upload-test-images.sh
@@ -16,15 +16,11 @@
 
 set -o errexit
 
-: ${1:?"Pass the directories with the test images as arguments"}
 : ${DOCKER_REPO_OVERRIDE:?"You must set 'DOCKER_REPO_OVERRIDE', see DEVELOPMENT.md"}
 
-DOCKER_FILES="$(find $@ -name Dockerfile)"
-: ${DOCKER_FILES:?"No subdirectories with Dockerfile files found in $@"}
+export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
+IMAGE_DIRS="$(find $(dirname $0)/test_images -mindepth 1 -maxdepth 1 -type d)"
 
-for docker_file in ${DOCKER_FILES}; do
-  image_dir="$(dirname ${docker_file})"
-  versioned_name="${DOCKER_REPO_OVERRIDE}/knative-serving/$(basename ${image_dir})"
-  docker build "${image_dir}" -f "${docker_file}" -t "${versioned_name}"
-  docker push "${versioned_name}"
+for image_dir in ${IMAGE_DIRS}; do
+  ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
 done