From 27ffc36dac3123c4933570b6ea8ea2e0d9422f1f Mon Sep 17 00:00:00 2001 From: Ajay Vohra Date: Mon, 9 Mar 2020 15:25:36 +0000 Subject: [PATCH] upgrade tensorpack, amis and fix issues in prepare script --- .../distributed-tensorflow/mask-rcnn-cfn.yaml | 40 +++++++++---------- .../prepare-s3-bucket.sh | 12 +++--- .../private-mask-rcnn-cfn.yaml | 40 +++++++++---------- examples/distributed-tensorflow/setup.sh | 30 ++++++++------ 4 files changed, 64 insertions(+), 58 deletions(-) diff --git a/examples/distributed-tensorflow/mask-rcnn-cfn.yaml b/examples/distributed-tensorflow/mask-rcnn-cfn.yaml index c7c8f97..d87000b 100644 --- a/examples/distributed-tensorflow/mask-rcnn-cfn.yaml +++ b/examples/distributed-tensorflow/mask-rcnn-cfn.yaml @@ -233,46 +233,46 @@ Conditions: Mappings: Ubuntu: us-east-1: - AMI: ami-0da5089baf511b65f + AMI: ami-025ed45832b817a35 us-east-2: - AMI: ami-0bd0eb312b384fbf0 + AMI: ami-0e30cdd8359d89531 us-west-2: - AMI: ami-04c7f6eb97b44200a + AMI: ami-07728e9e2742b0662 eu-west-1: - AMI: ami-06f483a626f873983 + AMI: ami-078d068af898f9114 eu-central-1: - AMI: ami-07f6e5544a1e9519d + AMI: ami-089c6ba0a6e0e7241 ap-southeast-1: - AMI: ami-02b27a073c4c1185a + AMI: ami-006124de1c99607fd ap-southeast-2: - AMI: ami-0f3691de8a64b4f59 + AMI: ami-08ec67632a3be867b ap-south-1: - AMI: ami-060245f7db2e15478 + AMI: ami-03aa5b53da3a8f3a5 ap-northeast-1: - AMI: ami-0e561552b0af89fb3 + AMI: ami-039d0c16bef5e0155 ap-northeast-2: - AMI: ami-00ba4cbabab783134 + AMI: ami-099a57eaf71294a34 AmazonLinux: us-east-1: - AMI: ami-00b484909a9bacd9f + AMI: ami-02bd97932dabc037b us-east-2: - AMI: ami-03670c2f79d09e4f5 + AMI: ami-00922dc6518dad1d9 us-west-2: - AMI: ami-05fc4809d75d5e4d7 + AMI: ami-00da9ade5fa7b8a23 eu-west-1: - AMI: ami-0c24b43903f18dba3 + AMI: ami-049c9ed0eb7c5df0b eu-central-1: - AMI: ami-0ce62d38b019a19de + AMI: ami-058f6bfe15b3c9624 ap-southeast-1: - AMI: ami-01347675719a240fe + AMI: ami-07b1a196227f36603 ap-southeast-2: - AMI: ami-05f962edaa876c658 + AMI: ami-0f15fae27825ac282 ap-south-1: - AMI: ami-0cabd6d5c247d76b1 + AMI: ami-082a3ef32d3851a0b ap-northeast-1: - AMI: ami-099a105d96ba2c05b + AMI: ami-07663a5b4e31e9963 ap-northeast-2: - AMI: ami-0b7c814f45c5dbcf2 + AMI: ami-0c71241962531d095 S3: us-east-1: URL: 'https://s3.amazonaws.com/' diff --git a/examples/distributed-tensorflow/prepare-s3-bucket.sh b/examples/distributed-tensorflow/prepare-s3-bucket.sh index 7e32f4a..24997fe 100755 --- a/examples/distributed-tensorflow/prepare-s3-bucket.sh +++ b/examples/distributed-tensorflow/prepare-s3-bucket.sh @@ -1,5 +1,7 @@ #!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + # Customize S3_BUCKET S3_BUCKET= @@ -37,14 +39,12 @@ tar -cvf $STAGE_DIR/coco-2017.tar --directory $STAGE_DIR data aws s3 cp $STAGE_DIR/coco-2017.tar s3://$S3_BUCKET/$S3_PREFIX/coco-2017.tar git clone https://github.com/tensorpack/tensorpack.git $STAGE_DIR/tensorpack -cd $STAGE_DIR/tensorpack && git fetch origin 26664c3f1d58ae029ea6c3ba0af6ae11900b1e55 -cd $STAGE_DIR/tensorpack && git reset --hard 26664c3f1d58ae029ea6c3ba0af6ae11900b1e55 +cd $STAGE_DIR/tensorpack && git fetch origin 4ac2e22b8f4f19949b2ce67b2fa37a926d4b5816 +cd $STAGE_DIR/tensorpack && git reset --hard 4ac2e22b8f4f19949b2ce67b2fa37a926d4b5816 tar -cvf $STAGE_DIR/tensorpack.tar --directory $STAGE_DIR tensorpack aws s3 cp $STAGE_DIR/tensorpack.tar s3://$S3_BUCKET/$S3_PREFIX/tensorpack.tar -aws s3 cp run.sh s3://$S3_BUCKET/$S3_PREFIX/run.sh -aws s3 cp setup.sh s3://$S3_BUCKET/$S3_PREFIX/setup.sh -aws s3 cp attach-fsx.sh s3://$S3_BUCKET/$S3_PREFIX/attach-fsx.sh -aws s3 cp cluster-health-check.sh s3://$S3_BUCKET/$S3_PREFIX/cluster-health-check.sh +aws s3 cp $DIR/run.sh s3://$S3_BUCKET/$S3_PREFIX/run.sh +aws s3 cp $DIR/setup.sh s3://$S3_BUCKET/$S3_PREFIX/setup.sh diff --git a/examples/distributed-tensorflow/private-mask-rcnn-cfn.yaml b/examples/distributed-tensorflow/private-mask-rcnn-cfn.yaml index 6441789..255b882 100755 --- a/examples/distributed-tensorflow/private-mask-rcnn-cfn.yaml +++ b/examples/distributed-tensorflow/private-mask-rcnn-cfn.yaml @@ -220,46 +220,46 @@ Conditions: Mappings: Ubuntu: us-east-1: - AMI: ami-0da5089baf511b65f + AMI: ami-025ed45832b817a35 us-east-2: - AMI: ami-0bd0eb312b384fbf0 + AMI: ami-0e30cdd8359d89531 us-west-2: - AMI: ami-04c7f6eb97b44200a + AMI: ami-07728e9e2742b0662 eu-west-1: - AMI: ami-06f483a626f873983 + AMI: ami-078d068af898f9114 eu-central-1: - AMI: ami-07f6e5544a1e9519d + AMI: ami-089c6ba0a6e0e7241 ap-southeast-1: - AMI: ami-02b27a073c4c1185a + AMI: ami-006124de1c99607fd ap-southeast-2: - AMI: ami-0f3691de8a64b4f59 + AMI: ami-08ec67632a3be867b ap-south-1: - AMI: ami-060245f7db2e15478 + AMI: ami-03aa5b53da3a8f3a5 ap-northeast-1: - AMI: ami-0e561552b0af89fb3 + AMI: ami-039d0c16bef5e0155 ap-northeast-2: - AMI: ami-00ba4cbabab783134 + AMI: ami-099a57eaf71294a34 AmazonLinux: us-east-1: - AMI: ami-00b484909a9bacd9f + AMI: ami-02bd97932dabc037b us-east-2: - AMI: ami-03670c2f79d09e4f5 + AMI: ami-00922dc6518dad1d9 us-west-2: - AMI: ami-05fc4809d75d5e4d7 + AMI: ami-00da9ade5fa7b8a23 eu-west-1: - AMI: ami-0c24b43903f18dba3 + AMI: ami-049c9ed0eb7c5df0b eu-central-1: - AMI: ami-0ce62d38b019a19de + AMI: ami-058f6bfe15b3c9624 ap-southeast-1: - AMI: ami-01347675719a240fe + AMI: ami-07b1a196227f36603 ap-southeast-2: - AMI: ami-05f962edaa876c658 + AMI: ami-0f15fae27825ac282 ap-south-1: - AMI: ami-0cabd6d5c247d76b1 + AMI: ami-082a3ef32d3851a0b ap-northeast-1: - AMI: ami-099a105d96ba2c05b + AMI: ami-07663a5b4e31e9963 ap-northeast-2: - AMI: ami-0b7c814f45c5dbcf2 + AMI: ami-0c71241962531d095 S3: us-east-1: URL: 'https://s3.amazonaws.com/' diff --git a/examples/distributed-tensorflow/setup.sh b/examples/distributed-tensorflow/setup.sh index 06ed69f..78ccc64 100644 --- a/examples/distributed-tensorflow/setup.sh +++ b/examples/distributed-tensorflow/setup.sh @@ -1,13 +1,19 @@ #!/bin/bash -pip install --upgrade pip -pip3 install ujson==1.35 -pip3 install opencv-python==4.1.0.25 -pip3 install Cython==0.28.4 -pip3 install pycocotools==2.0.0 -pip3 install matplotlib==3.0.3 -pip3 install markdown==3.1 -pip install --ignore-installed -e $HOME/tensorpack -pip3 uninstall -y numpy -pip3 uninstall -y numpy -pip3 uninstall -y numpy -pip3 install numpy==1.17.0 + +pip uninstall -y numpy +pip install numpy==1.17.5 +pip install awscli +pip install boto3 +pip install ujson==1.35 +pip install opencv-python==4.1.0.25 +pip install Cython==0.28.4 +pip uninstall -y pycocotools +git clone https://github.com/cocodataset/cocoapi.git $HOME/cocoapi +cd $HOME/cocoapi && git fetch origin 8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9 +cd $HOME/cocoapi && git reset --hard 8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9 +cd $HOME/cocoapi/PythonAPI && make +pip install -e $HOME/cocoapi/PythonAPI +pip install matplotlib==3.0.3 +pip install markdown==3.1 + +pip install -e $HOME/tensorpack