Merge pull request #126 from NervanaSystems/ci_updates

CI related updates
IntelLabs · Nov 27, 2018 · e877920 · e877920
2 parents 4e0d018 + 3601d9b
commit e877920
Show file tree

Hide file tree

Showing 9 changed files with 20 additions and 313 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,15 +23,18 @@ aliases:
       name: Prepare aws cli
       command: |
         sudo pip install awscli pytest kubernetes==8.0.0b1
-        export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-        export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
 
         $(aws ecr get-login --no-include-email --region us-west-2)
         sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator
         sudo chmod a+x /usr/local/bin/aws-iam-authenticator
         aws eks update-kubeconfig --name coach-aws-cicd
         sudo curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
         sudo chmod a+x /usr/local/bin/kubectl
+  - &docker_prep
+    run:
+      name: Prepare dockerfiles
+      command: |
+        aws s3 cp --recursive s3://coach-ci-dockerfiles docker
 
 version: 2
 jobs:
@@ -42,6 +45,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push base and main container
           command: |
@@ -72,6 +76,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push gym environment container
           command: |
@@ -90,6 +95,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push vizdoom environment container
           command: |
@@ -108,6 +114,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Build and push mujoco environment container
           command: |
@@ -126,17 +133,14 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run unit tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             python3 rl_coach/tests/test_eks.py  -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn unit-test -tc 'make unit_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=unit-test-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns unit-test-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -148,17 +152,14 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run integration tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             python3 rl_coach/tests/test_eks.py  -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn integration-test -tc 'make integration_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=integration-test-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns integration-test-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -170,19 +171,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run gym related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-gym -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-gym-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-gym-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -194,19 +192,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run doom related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-doom -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-doom-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-doom-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -218,19 +213,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run mujoco related golden tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-mujoco -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -242,19 +234,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run gym related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-gym-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-gym-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -266,19 +255,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run doom related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-doom-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-doom-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -290,19 +276,16 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: run mujoco related trace tests
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
           name: cleanup
           command: |
-            export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
-            export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
             kubectl delete --all pods --namespace=trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
             kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
@@ -314,6 +297,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *aws_prep
+      - *docker_prep
       - run:
           name: Tag and push updated base and main container
           command: |
@@ -393,4 +377,4 @@ workflows:
             - integration_tests
           filters:
             branches:
-              only: 0.11.0-release
+              only: master
diff --git a/README.md b/README.md
@@ -1,5 +1,6 @@
 # Coach
 
+[![CircleCI](https://circleci.com/gh/NervanaSystems/coach/tree/master.svg?style=svg)](https://circleci.com/gh/NervanaSystems/coach/tree/master)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/NervanaSystems/coach/blob/master/LICENSE)
 [![Docs](https://readthedocs.org/projects/carla/badge/?version=latest)](https://nervanasystems.github.io/coach/)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1134898.svg)](https://doi.org/10.5281/zenodo.1134898)

diff --git a/docker/Dockerfile b/docker/Dockerfile
diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
diff --git a/docker/Dockerfile.doom_environment b/docker/Dockerfile.doom_environment
diff --git a/docker/Dockerfile.gym_environment b/docker/Dockerfile.gym_environment
diff --git a/docker/Dockerfile.mujoco_environment b/docker/Dockerfile.mujoco_environment