From 0e408534002a30e1adac95f2f6d93830379c44a6 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 12 Jun 2018 11:00:14 -0700 Subject: [PATCH 1/5] Pushing to s3 --- python/asv.conf.json | 2 +- test/jenkins_tests/run_asv.sh | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/python/asv.conf.json b/python/asv.conf.json index 0378ba0da2ab..2436e77863b5 100644 --- a/python/asv.conf.json +++ b/python/asv.conf.json @@ -101,7 +101,7 @@ // The directory (relative to the current directory) that raw benchmark // results are stored in. If not provided, defaults to "results". - // "results_dir": "results", + "results_dir": "ASV_RESULTS", // The directory (relative to the current directory) that the html tree // should be written to. If not provided, defaults to "html". diff --git a/test/jenkins_tests/run_asv.sh b/test/jenkins_tests/run_asv.sh index 108826b0a4e9..f6e1d2c95b23 100755 --- a/test/jenkins_tests/run_asv.sh +++ b/test/jenkins_tests/run_asv.sh @@ -6,10 +6,22 @@ set -e # Show explicitly which commands are currently running. set -x +BUCKET_NAME=raylogs +COMMIT=$(cat /ray/git-rev) +ASV_RESULTS_DIR=/ray/python/ASV_RESULTS +pip install awscli + +# Install Ray fork of ASV git clone https://github.com/ray-project/asv.git /tmp/asv/ || true cd /tmp/asv/ pip install -e . cd /ray/python/ -asv machine --yes -asv run --show-stderr --python=same --force-record-commit=$(cat ../git-rev) +asv machine --machine jenkins +mkdir $ASV_RESULTS_DIR || true +aws s3 cp s3://$BUCKET_NAME/ASV_RESULTS/benchmarks.json $ASV_RESULTS_DIR/benchmarks.json || true + +asv run --show-stderr --python=same --force-record-commit=$COMMIT + +aws s3 cp $ASV_RESULTS_DIR/benchmarks.json s3://$BUCKET_NAME/ASV_RESULTS/benchmarks_$COMMIT.json +aws s3 sync $ASV_RESULTS_DIR/ s3://$BUCKET_NAME/ASV_RESULTS/ From 46c07f3de7b8d060521a5236b3a3b9a8d0a8cac4 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 12 Jun 2018 12:08:47 -0700 Subject: [PATCH 2/5] fix for publishing --- python/asv.conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/asv.conf.json b/python/asv.conf.json index 2436e77863b5..b21f004b3c4a 100644 --- a/python/asv.conf.json +++ b/python/asv.conf.json @@ -11,7 +11,7 @@ // The URL or local path of the source code repository for the // project being benchmarked - "repo": ".", + "repo": "../", // List of branches to benchmark. If not provided, defaults to "master" // (for git) or "default" (for mercurial). From ab629c413cf978de5eab19d79ac4df371fb1ee4d Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 12 Jun 2018 13:00:56 -0700 Subject: [PATCH 3/5] new dir for bucket --- python/README-benchmarks.rst | 19 ++++++++++++++++++- test/jenkins_tests/run_asv.sh | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/python/README-benchmarks.rst b/python/README-benchmarks.rst index aa1acca38f2d..9a0a07aab4ba 100644 --- a/python/README-benchmarks.rst +++ b/python/README-benchmarks.rst @@ -7,4 +7,21 @@ You can run the benchmark suite by doing the following: 2. Run ``asv dev`` in this directory. To run ASV inside docker, you can use the following command: -`docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'` +``docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'`` + +Visualizing Benchmarks +====================== + +To visualize benchmarks, you must copy the S3 bucket down to `$RAY_DIR/python`. Assuming asv is installed, + +``` +cd $RAY_DIR/python +aws s3 sync s3://BUCKET/ . +``` +Then, you can run: +``` +asv publish --no-pull +asv preview +``` + +This creates diff --git a/test/jenkins_tests/run_asv.sh b/test/jenkins_tests/run_asv.sh index f6e1d2c95b23..0df1710d2579 100755 --- a/test/jenkins_tests/run_asv.sh +++ b/test/jenkins_tests/run_asv.sh @@ -6,7 +6,7 @@ set -e # Show explicitly which commands are currently running. set -x -BUCKET_NAME=raylogs +BUCKET_NAME=ray-integration-testing/ASV COMMIT=$(cat /ray/git-rev) ASV_RESULTS_DIR=/ray/python/ASV_RESULTS pip install awscli From d1e74cb473e356ae6fee2cecf7a8efaa45255535 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 12 Jun 2018 15:59:54 -0700 Subject: [PATCH 4/5] readme --- python/README-benchmarks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/README-benchmarks.rst b/python/README-benchmarks.rst index 9a0a07aab4ba..18d5107cf1c9 100644 --- a/python/README-benchmarks.rst +++ b/python/README-benchmarks.rst @@ -16,7 +16,7 @@ To visualize benchmarks, you must copy the S3 bucket down to `$RAY_DIR/python`. ``` cd $RAY_DIR/python -aws s3 sync s3://BUCKET/ . +aws s3 sync s3://$BUCKET/ASV/ . ``` Then, you can run: ``` @@ -24,4 +24,4 @@ asv publish --no-pull asv preview ``` -This creates +This creates the directory and then launches a server. From 49cf796664f681a912d1e69db350ff669a857e1f Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 12 Jun 2018 17:50:36 -0700 Subject: [PATCH 5/5] update readme --- python/README-benchmarks.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/README-benchmarks.rst b/python/README-benchmarks.rst index 18d5107cf1c9..2837fc2967d4 100644 --- a/python/README-benchmarks.rst +++ b/python/README-benchmarks.rst @@ -14,14 +14,16 @@ Visualizing Benchmarks To visualize benchmarks, you must copy the S3 bucket down to `$RAY_DIR/python`. Assuming asv is installed, -``` -cd $RAY_DIR/python -aws s3 sync s3://$BUCKET/ASV/ . -``` +.. code-block:: + + cd $RAY_DIR/python + aws s3 sync s3://$BUCKET/ASV/ . + Then, you can run: -``` -asv publish --no-pull -asv preview -``` + +.. code-block:: + + asv publish --no-pull + asv preview This creates the directory and then launches a server.