diff --git a/.circleci/config.yml b/.circleci/config.yml
index c6220fce9..96d51b657 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -23,8 +23,6 @@ aliases:
       name: Prepare aws cli
       command: |
         sudo pip install awscli pytest kubernetes==8.0.0b1
-        export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-        export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
 
         $(aws ecr get-login --no-include-email --region us-west-2)
         sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator
@@ -32,6 +30,11 @@ aliases:
         aws eks update-kubeconfig --name coach-aws-cicd
         sudo curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
         sudo chmod a+x /usr/local/bin/kubectl
+  - &docker_prep
+    run:
+      name: Prepare dockerfiles
+      command: |
+        aws s3 cp --recursive s3://coach-ci-dockerfiles docker
 
 version: 2
 jobs:
@@ -42,6 +45,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push base and main container
           command: |
@@ -72,6 +76,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push gym environment container
           command: |
@@ -90,6 +95,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push vizdoom environment container
           command: |
@@ -108,6 +114,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push mujoco environment container
           command: |
@@ -126,17 +133,14 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run unit tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             python3 rl_coach/tests/test_eks.py  -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn unit-test -tc 'make unit_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=unit-test-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns unit-test-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -148,17 +152,14 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run integration tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             python3 rl_coach/tests/test_eks.py  -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn integration-test -tc 'make integration_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=integration-test-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns integration-test-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -170,19 +171,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run gym related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-gym -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-gym-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-gym-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -194,19 +192,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run doom related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-doom -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-doom-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-doom-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -218,19 +213,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run mujoco related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-mujoco -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -242,19 +234,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run gym related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-gym-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-gym-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -266,19 +255,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run doom related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-doom-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-doom-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -290,19 +276,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run mujoco related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -314,6 +297,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Tag and push updated base and main container
           command: |
@@ -393,4 +377,4 @@ workflows:
             - integration_tests
           filters:
             branches:
-              only: 0.11.0-release
+              only: master
diff --git a/README.md b/README.md
index b341cbb0f..b98799006 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
 # Coach
 
+[![CircleCI](https://circleci.com/gh/NervanaSystems/coach/tree/master.svg?style=svg)](https://circleci.com/gh/NervanaSystems/coach/tree/master)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/NervanaSystems/coach/blob/master/LICENSE)
 [![Docs](https://readthedocs.org/projects/carla/badge/?version=latest)](https://nervanasystems.github.io/coach/)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1134898.svg)](https://doi.org/10.5281/zenodo.1134898)
diff --git a/docker/Dockerfile b/docker/Dockerfile
deleted file mode 100644
index 6ac66d06d..000000000
--- a/docker/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-FROM coach-base:master as builder
-
-# prep some of the more common environments
-# Gym (installed with coach)
-Run pip3 install gym[atari] box2d
-# Mujoco
-RUN mkdir -p ~/.mujoco \
-    && wget https://www.roboti.us/download/mjpro150_linux.zip -O mujoco.zip \
-    && unzip mujoco.zip -d ~/.mujoco \
-    && rm mujoco.zip
-ARG MUJOCO_KEY
-ENV MUJOCO_KEY=$MUJOCO_KEY
-ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:$LD_LIBRARY_PATH
-RUN echo $MUJOCO_KEY | base64 --decode > /root/.mujoco/mjkey.txt
-RUN pip3 install mujoco_py
-# Vizdoom
-RUN pip3 install vizdoom
-
-RUN mkdir /root/src
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-RUN pip3 install -r /root/src/requirements.txt
-
-FROM coach-base:master
-WORKDIR /root/src
-COPY --from=builder /root/.mujoco /root/.mujoco
-ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:$LD_LIBRARY_PATH
-COPY --from=builder /root/.cache /root/.cache
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-COPY README.md /root/src/.
-RUN pip3 install gym[atari] box2d mujoco_py vizdoom && pip3 install -e .[all] && rm -rf /root/.cache
-COPY . /root/src
diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
deleted file mode 100644
index 75cc6157b..000000000
--- a/docker/Dockerfile.base
+++ /dev/null
@@ -1,39 +0,0 @@
-FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
-
-################################
-# Install apt-get Requirements #
-################################
-
-# General
-RUN apt-get update && \
-    apt-get install -y python3-pip cmake zlib1g-dev python3-tk python-opencv \
-    # Boost libraries
-    libboost-all-dev \
-    # Scipy requirements
-    libblas-dev liblapack-dev libatlas-base-dev gfortran \
-    # Pygame requirements
-    libsdl-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev \
-    libsmpeg-dev libportmidi-dev libavformat-dev libswscale-dev \
-    # Dashboard
-    dpkg-dev build-essential python3.5-dev libjpeg-dev  libtiff-dev libsdl1.2-dev libnotify-dev \
-    freeglut3 freeglut3-dev libsm-dev libgtk2.0-dev libgtk-3-dev libwebkitgtk-dev libgtk-3-dev \
-    libwebkitgtk-3.0-dev libgstreamer-plugins-base1.0-dev \
-    # Gym
-    libav-tools libsdl2-dev swig cmake \
-    # Mujoco_py
-    curl libgl1-mesa-dev libgl1-mesa-glx libglew-dev libosmesa6-dev software-properties-common \
-    # ViZDoom
-    build-essential zlib1g-dev libsdl2-dev libjpeg-dev \
-    nasm tar libbz2-dev libgtk2.0-dev cmake git libfluidsynth-dev libgme-dev \
-    libopenal-dev timidity libwildmidi-dev unzip wget && \
-    apt-get clean autoclean && \
-    apt-get autoremove -y
-
-############################
-# Install Pip Requirements #
-############################
-RUN pip3 install --upgrade pip
-RUN pip3 install setuptools==39.1.0 && pip3 install pytest && pip3 install pytest-xdist
-
-RUN curl -o /usr/local/bin/patchelf https://s3-us-west-2.amazonaws.com/openai-sci-artifacts/manual-builds/patchelf_0.9_amd64.elf \
-    && chmod +x /usr/local/bin/patchelf
diff --git a/docker/Dockerfile.doom_environment b/docker/Dockerfile.doom_environment
deleted file mode 100644
index 4498e80d5..000000000
--- a/docker/Dockerfile.doom_environment
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM coach-base:master as builder
-
-# prep vizdoom and any of its related requirements.
-RUN pip3 install vizdoom
-
-# add coach source starting with files that could trigger
-# re-build if dependencies change.
-RUN mkdir /root/src
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-RUN pip3 install -r /root/src/requirements.txt
-
-FROM coach-base:master
-WORKDIR /root/src
-COPY --from=builder /root/.cache /root/.cache
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-COPY README.md /root/src/.
-RUN pip3 install vizdoom && pip3 install -e .[all] && rm -rf /root/.cache
-COPY . /root/src
diff --git a/docker/Dockerfile.gym_environment b/docker/Dockerfile.gym_environment
deleted file mode 100644
index f66779871..000000000
--- a/docker/Dockerfile.gym_environment
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM coach-base:master as builder
-
-# prep gym and any of its related requirements.
-RUN pip3 install gym[atari,box2d,classic_control]==0.10.5
-
-# add coach source starting with files that could trigger
-# re-build if dependencies change.
-RUN mkdir /root/src
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-RUN pip3 install -r /root/src/requirements.txt
-
-FROM coach-base:master
-WORKDIR /root/src
-COPY --from=builder /root/.cache /root/.cache
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-COPY README.md /root/src/.
-RUN pip3 install gym[atari,box2d,classic_control]==0.10.5 && pip3 install -e .[all] && rm -rf /root/.cache
-COPY . /root/src
diff --git a/docker/Dockerfile.mujoco_environment b/docker/Dockerfile.mujoco_environment
deleted file mode 100644
index 195928310..000000000
--- a/docker/Dockerfile.mujoco_environment
+++ /dev/null
@@ -1,31 +0,0 @@
-FROM coach-base:master as builder
-
-# prep mujoco and any of its related requirements.
-# Mujoco
-RUN mkdir -p ~/.mujoco \
-    && wget https://www.roboti.us/download/mjpro150_linux.zip -O mujoco.zip \
-    && unzip -n mujoco.zip -d ~/.mujoco \
-    && rm mujoco.zip
-ARG MUJOCO_KEY
-ENV MUJOCO_KEY=$MUJOCO_KEY
-ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:$LD_LIBRARY_PATH
-RUN echo $MUJOCO_KEY | base64 --decode > /root/.mujoco/mjkey.txt
-RUN pip3 install mujoco_py
-
-# add coach source starting with files that could trigger
-# re-build if dependencies change.
-RUN mkdir /root/src
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-RUN pip3 install -r /root/src/requirements.txt
-
-FROM coach-base:master
-WORKDIR /root/src
-COPY --from=builder /root/.mujoco /root/.mujoco
-ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:$LD_LIBRARY_PATH
-COPY --from=builder /root/.cache /root/.cache
-COPY setup.py /root/src/.
-COPY requirements.txt /root/src/.
-COPY README.md /root/src/.
-RUN pip3 install mujoco_py && pip3 install -e .[all] && rm -rf /root/.cache
-COPY . /root/src
diff --git a/docker/Makefile b/docker/Makefile
deleted file mode 100644
index 757a3077c..000000000
--- a/docker/Makefile
+++ /dev/null
@@ -1,108 +0,0 @@
-# REGISTRY=gcr.io
-REGISTRY=docker.io
-ORGANIZATION=nervana
-IMAGE=coach
-CONTEXT = $(realpath ..)
-
-BUILD_ARGUMENTS=
-RUN_ARGUMENTS=
-ifdef http_proxy
-	BUILD_ARGUMENTS+=--build-arg http_proxy=$(http_proxy)
-	RUN_ARGUMENTS+=--env http_proxy=$(http_proxy)
-endif
-
-ifdef https_proxy
-	BUILD_ARGUMENTS+=--build-arg https_proxy=$(https_proxy)
-	RUN_ARGUMENTS+=--env https_proxy=$(https_proxy)
-endif
-
-ifndef MUJOCO_KEY
-KEY_NAME=${CONTEXT}/mjkey.txt
-ifneq ("$(wildcard ${KEY_NAME})", "")
-MUJOCO_KEY:=$(shell cat ${KEY_NAME} | base64 -w 0)
-endif
-endif
-
-ifdef MUJOCO_KEY
-	BUILD_ARGUMENTS+=--build-arg MUJOCO_KEY=$(MUJOCO_KEY)
-endif
-
-RUN_ARGUMENTS+=--rm
-RUN_ARGUMENTS+=--net host
-RUN_ARGUMENTS+=-v /tmp/checkpoint:/checkpoint
-
-UNIT_TESTS=python3 -m pytest rl_coach/tests -m unit_test
-INTEGRATION_TESTS=python3 -m pytest rl_coach/tests -m integration_test --tb=short
-ifdef PRESETS
-	PRESETS := -p $(PRESETS)
-else
-	PRESETS :=
-endif
-GOLDEN_TESTS=python3 rl_coach/tests/test_golden.py ${PRESETS}
-TRACE_TESTS=python3 rl_coach/tests/trace_tests.py -prl ${PRESETS}
-
-ifndef DOCKER
-	DOCKER = docker
-endif
-
-build_base:
-	${DOCKER} build -f=Dockerfile.base -t=${IMAGE}-base:master ${BUILD_ARGUMENTS} ${CONTEXT}
-
-build: build_base
-	${DOCKER} build -f=Dockerfile -t=${IMAGE} ${BUILD_ARGUMENTS} ${CONTEXT}
-	mkdir -p /tmp/checkpoint
-	rm -rf /tmp/checkpoint/*
-
-shell: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} /bin/bash
-
-unit_tests: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} ${UNIT_TESTS}
-
-integration_tests: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} ${INTEGRATION_TESTS}
-
-golden_tests: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} ${GOLDEN_TESTS} ${PRESETS}
-
-trace_tests: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} ${TRACE_TESTS}
-
-run: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE}
-
-run_training_worker: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} python3 rl_coach/training_worker.py --preset CartPole_DQN_distributed
-
-run_rollout_worker: build
-	${DOCKER} run ${RUN_ARGUMENTS} -it ${IMAGE} python3 rl_coach/rollout_worker.py --preset CartPole_DQN_distributed
-
-bootstrap_kubernetes: build push
-	kubectl run -i --tty --attach --image=${REGISTRY}/${IMAGE} --restart=Never distributed-coach -- python3 rl_coach/orchestrators/start_training.py --preset CartPole_DQN_distributed --image ${IMAGE} -ns 10.63.249.182 -np /
-
-stop_kubernetes:
-	kubectl delete service --ignore-not-found redis-service
-	kubectl delete pv --ignore-not-found nfs-checkpoint-pv
-	kubectl delete pvc --ignore-not-found nfs-checkpoint-pvc
-	kubectl delete deployment --ignore-not-found redis-server
-	kubectl get jobs | grep train | awk "{print $\1}" | xargs kubectl delete jobs
-	kubectl get jobs | grep worker | awk "{print $\1}" | xargs kubectl delete jobs
-
-kubernetes: stop_kubernetes
-	python3 ${CONTEXT}/rl_coach/orchestrators/start_training.py --preset CartPole_DQN_distributed --image ${IMAGE} -ns 10.63.249.182 -np /
-
-push: build
-	${DOCKER} tag ${IMAGE} ${REGISTRY}/${ORGANIZATION}/${IMAGE}
-	${DOCKER} push ${REGISTRY}/${ORGANIZATION}/${IMAGE}
-
-unit_tests_without_docker:
-	cd .. && ${UNIT_TESTS}
-
-integration_tests_without_docker:
-	cd .. && ${INTEGRATION_TESTS}
-
-golden_tests_without_docker:
-	cd .. && ${GOLDEN_TESTS}
-
-trace_tests_without_docker:
-	cd .. && ${TRACE_TESTS}
diff --git a/docker/README.md b/docker/README.md
deleted file mode 100644
index b9a8e8f4d..000000000
--- a/docker/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-# Container Images
-
-In this directory we've put together several different Dockerfile's that can be used to build
-containers that have coach and other environments/dependencies installed.  How to build these
-and what each contains is defined below:
-
-## default `Dockerfile`
-* `make build` to create the image
-* will create a basic Coach installation along with Gym (atari), Mujoco, and Vizdoom environments.
-* useful for running unit/integration tests  `make unit_tests` to run these in the container
-* `make shell` will launch this container locally, and provide a bash shell prompt.
-* includes GPU support (derives from `Dockerfile.base` which is a CUDA ubuntu 16.04 derived image)
-
-## `Dockerfile.mujoco_environment`
-* `docker build --build-arg MUJOCO_KEY=${MUJOCO_KEY} -f docker/Dockerfile.mujoco_environment .`
-  from the parent dir to create the image
-* contains mujoco environment and Coach.
-* you need to supply your own license key (base64 encrypted) as an environment variable `MUJOCO_KEY`
-  to ensure you get the complete Mujoco environment
-
-## `Dockerfile.gym_environment`
-* `docker build -f docker/Dockerfile.gym_environment .` from the parent dir to create the image
-* contains OpenAI Gym environment (and all extras) and Coach.
-
-## `Dockerfile.doom_environment`
-* `docker build -f docker/Dockerfile.doom_environment .` from the parent dir to create the image
-* contains vizdoom environment and Coach.