Skip to content

Commit a0ad6a7

Browse files
muupanprabhatnagarajanummavimarioyckeisuke-nakata
committed
Initial commit
Co-authored-by: Prabhat Nagarajan <[email protected]> Co-authored-by: Avinash Ummadisingu <[email protected]> Co-authored-by: Mario Ynocente Castro <[email protected]> Co-authored-by: Nakata Keisuke <[email protected]> Co-authored-by: Toshiki Kataoka <[email protected]>
0 parents  commit a0ad6a7

File tree

312 files changed

+31561
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

312 files changed

+31561
-0
lines changed

‎.flake8

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[flake8]
2+
max-line-length = 88
3+
extend-ignore = E203, E231, W503

‎.gitignore

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
*.pyc
2+
.ipynb_checkpoints
3+
pfrl.egg-info
4+
build/
5+
dist/
6+
.idea/
7+
results/
8+
examples/gym/results/

‎.pfnci/config.pbtxt

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
configs {
2+
key: "pfrl.gpu"
3+
value {
4+
requirement {
5+
cpu: 10
6+
memory: 30
7+
gpu: 1
8+
}
9+
time_limit {
10+
seconds: 1200
11+
}
12+
environment_variables { key: "GPU" value: "1" }
13+
command: "bash .pfnci/script.sh gpu"
14+
}
15+
}
16+
17+
configs {
18+
key: "pfrl.gpu.slow"
19+
value {
20+
requirement {
21+
cpu: 10
22+
memory: 30
23+
gpu: 1
24+
}
25+
time_limit {
26+
seconds: 2400
27+
}
28+
environment_variables { key: "GPU" value: "1" }
29+
environment_variables { key: "SLOW" value: "1" }
30+
command: "bash .pfnci/script.sh gpu"
31+
}
32+
}
33+
34+
configs {
35+
key: "pfrl.gpu.examples"
36+
value {
37+
requirement {
38+
cpu: 2
39+
memory: 12
40+
gpu: 1
41+
}
42+
time_limit {
43+
seconds: 1200
44+
}
45+
environment_variables { key: "GPU" value: "1" }
46+
environment_variables { key: "TEST_EXAMPLES" value: "1" }
47+
command: "bash .pfnci/script.sh gpu"
48+
}
49+
}
50+
51+
configs {
52+
key: "pfrl.cpu.slow"
53+
value {
54+
requirement {
55+
cpu: 10
56+
memory: 30
57+
}
58+
time_limit {
59+
seconds: 2400
60+
}
61+
environment_variables { key: "SLOW" value: "1" }
62+
command: "bash .pfnci/script.sh cpu"
63+
}
64+
}
65+
66+
# PFRL CPU-only unit tests.
67+
configs {
68+
key: "pfrl.cpu"
69+
value {
70+
requirement {
71+
cpu: 10
72+
memory: 30
73+
}
74+
time_limit {
75+
seconds: 1200
76+
}
77+
command: "bash .pfnci/script.sh cpu"
78+
}
79+
}
80+
81+
configs {
82+
key: "pfrl.cpu.examples"
83+
value {
84+
requirement {
85+
cpu: 2
86+
memory: 12
87+
}
88+
time_limit {
89+
seconds: 1200
90+
}
91+
environment_variables { key: "TEST_EXAMPLES" value: "1" }
92+
command: "bash .pfnci/script.sh cpu"
93+
}
94+
}
95+
96+
# static code check
97+
configs {
98+
key: "pfrl.lint"
99+
value {
100+
requirement {
101+
cpu: 1
102+
memory: 1
103+
}
104+
time_limit {
105+
seconds: 1200
106+
}
107+
command: "bash .pfnci/lint.sh"
108+
}
109+
}

‎.pfnci/hint.pbtxt

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# hint.pbtxt is a config file for xpytest.
2+
#
3+
# Proto type: xpytest.proto.HintFile
4+
# https://github.com/chainer/xpytest/blob/master/proto/test_case.proto
5+
6+
# Slow tests take 60+ seconds.
7+
rules { name: "agents_tests/test_ddpg.py" xdist: 4 deadline: 600 }
8+
rules { name: "agents_tests/test_reinforce.py" xdist: 4 deadline: 480 }
9+
rules { name: "agents_tests/test_dqn.py" xdist: 4 deadline: 480 }
10+
11+
rules { name: "agents_tests/test_a2c.py" xdist: 4 deadline: 240 }
12+
rules { name: "agents_tests/test_al.py" xdist: 4 deadline: 240 }
13+
rules { name: "agents_tests/test_categorical_dqn.py" xdist: 4 deadline: 240 }
14+
rules { name: "agents_tests/test_double_categorical_dqn.py" xdist: 4 deadline: 240 }
15+
rules { name: "agents_tests/test_double_dqn.py" xdist: 4 deadline: 240 }
16+
rules { name: "agents_tests/test_double_iqn.py" xdist: 4 deadline: 240 }
17+
rules { name: "agents_tests/test_double_pal.py" xdist: 4 deadline: 240 }
18+
rules { name: "agents_tests/test_dpp.py" xdist: 4 deadline: 240 }
19+
rules { name: "agents_tests/test_iqn.py" xdist: 4 deadline: 240 }
20+
rules { name: "agents_tests/test_ppo.py" xdist: 4 deadline: 240 }
21+
rules { name: "agents_tests/test_pal.py" xdist: 4 deadline: 240 }
22+
rules { name: "agents_tests/test_trpo.py" xdist: 4 deadline: 240 }
23+
24+
rules { name: "agents_tests/test_a3c.py" xdist: 4 deadline: 240 }
25+
rules { name: "agents_tests/test_agents.py" xdist: 4 deadline: 240 }
26+
rules { name: "agents_tests/test_soft_actor_critic.py" xdist: 4 deadline: 240 }
27+
rules { name: "agents_tests/test_td3.py" xdist: 4 deadline: 240 }
28+
rules { name: "utils_tests/test_random.py" xdist: 4 deadline: 240 }
29+
30+
# Slow tests take 10+ seconds.
31+
rules { name: "agents_tests/test_acer.py" }
32+
rules { name: "agents_tests/test_ale.py" }
33+
rules { name: "agents_tests/test_trpo.py" }
34+
rules { name: "policies_tests/test_deterministic_policy.py" }
35+
rules { name: "q_functions_tests/test_state_action_q_function.py" }
36+
rules { name: "tests/test_ale.py" }

‎.pfnci/lint.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
# Use latest black to apply https://github.com/psf/black/issues/1288
6+
pip3 install git+git://github.com/psf/black.git@88d12f88a97e5e4c8fd0d245df0a311e932fd1e1 flake8 mypy
7+
8+
black --diff --check pfrl tests examples
9+
flake8 pfrl tests examples
10+
mypy pfrl
11+
# mypy does not search child directories unless there is __init__.py
12+
find tests -type f -name "*.py" | xargs dirname | sort | uniq | xargs mypy
13+
find examples -type f -name "*.py" | xargs dirname | sort | uniq | xargs mypy

‎.pfnci/run.sh

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/bin/bash
2+
# run.sh is a script to run unit tests inside Docker. This should be injected
3+
# by and called from script.sh.
4+
#
5+
# Usage: run.sh [target]
6+
# - target is a test target (e.g., "py37").
7+
#
8+
# Environment variables:
9+
# - GPU (default: 0) ... Set a number of GPUs to GPU.
10+
# CAVEAT: Setting GPU>0 disables non-GPU tests, and setting GPU=0 disables
11+
# GPU tests.
12+
13+
set -eux
14+
15+
cp -a /src /pfrl
16+
mkdir -p /pfrl/.git
17+
cd /
18+
19+
# Remove pyc files. When the CI is triggered with a user's local files, pyc
20+
# files generated on the user's local machine and they often cause failures.
21+
find /pfrl -name "*.pyc" -exec rm -f {} \;
22+
23+
TARGET="$1"
24+
: "${GPU:=0}"
25+
: "${XPYTEST_NUM_THREADS:=$(nproc)}"
26+
: "${PYTHON=python3}"
27+
: "${SLOW:=0}"
28+
: "${TEST_EXAMPLES:=0}"
29+
30+
# Use multi-process service to prevent GPU flakiness caused by running many
31+
# processes on a GPU. Specifically, it seems running more than 16 processes
32+
# sometimes causes "cudaErrorLaunchFailure: unspecified launch failure".
33+
if (( GPU > 0 )); then
34+
nvidia-smi -c EXCLUSIVE_PROCESS
35+
nvidia-cuda-mps-control -d
36+
fi
37+
38+
################################################################################
39+
# Main function
40+
################################################################################
41+
42+
main() {
43+
if (( !SLOW )); then
44+
marker='not slow'
45+
else
46+
marker='slow'
47+
fi
48+
if (( !GPU )); then
49+
marker+=' and not gpu'
50+
bucket=1
51+
else
52+
marker+=' and gpu'
53+
bucket="${GPU}"
54+
fi
55+
marker+=' and not download_model'
56+
57+
58+
UBUNTU_VERSION_ID=$(grep DISTRIB_RELEASE /etc/lsb-release | cut -d "=" -f2)
59+
if [ "$UBUNTU_VERSION_ID" = "16.04" ]; then
60+
# Because ffmpeg of ubuntu 16.04 causes segmentation fault,
61+
# we use jonathonf/ffmpeg-3
62+
apt-get update -q
63+
apt-get install -qy --no-install-recommends software-properties-common
64+
add-apt-repository ppa:cran/ffmpeg-3
65+
fi
66+
67+
apt-get update -q
68+
apt-get install -qy --no-install-recommends \
69+
"${PYTHON}-dev" "${PYTHON}-pip" "${PYTHON}-setuptools" \
70+
zlib1g-dev make cmake g++ git ffmpeg freeglut3-dev xvfb
71+
72+
"${PYTHON}" -m pip install scipy==1.4.1
73+
74+
"${PYTHON}" -m pip install /pfrl
75+
# TODO(pfrl): Prepare test target instead.
76+
# pytest does not run with attrs==19.2.0 (https://github.com/pytest-dev/pytest/issues/3280) # NOQA
77+
"${PYTHON}" -m pip install \
78+
'pytest==4.1.1' 'attrs==19.1.0' 'pytest-xdist==1.26.1' \
79+
'atari_py==0.1.1' 'opencv-python' 'zipp==1.0.0' 'pybullet==2.8.1'
80+
81+
git config --global user.email "[email protected]"
82+
git config --global user.name "Your Name"
83+
84+
if [ "$TEST_EXAMPLES" = "1" ]; then
85+
(
86+
cd /pfrl
87+
# export python3 as python because `python` is used in test_examples.sh
88+
export -f python
89+
if (( !GPU )); then
90+
bash test_examples.sh -1
91+
else
92+
bash test_examples.sh 0
93+
fi
94+
)
95+
else
96+
xpytest_args=(
97+
--python="${PYTHON}" -m "${marker}"
98+
--bucket="${bucket}" --thread="$(( XPYTEST_NUM_THREADS / bucket ))"
99+
--hint="/pfrl/.pfnci/hint.pbtxt"
100+
)
101+
# Xvfb's default screen is 1280x1024x8, which seems to cause a problem.
102+
# https://bugzilla.redhat.com/show_bug.cgi?id=904851
103+
OMP_NUM_THREADS=1 PYTHONHASHSEED=0 \
104+
xvfb-run --server-args="-screen 0 1280x800x24" \
105+
xpytest "${xpytest_args[@]}" -m "${marker} and not async_" '/pfrl/tests/**/test_*.py'
106+
# Async training is tested in a separate process to work around https://github.com/pytorch/pytorch/wiki/Autograd-and-Fork
107+
OMP_NUM_THREADS=1 PYTHONHASHSEED=0 \
108+
xvfb-run --server-args="-screen 0 1280x800x24" \
109+
xpytest "${xpytest_args[@]}" -m "${marker} and async_" '/pfrl/tests/**/test_*.py'
110+
fi
111+
112+
113+
}
114+
115+
python() {
116+
python3 "$@"
117+
}
118+
119+
main

‎.pfnci/script.sh

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/bin/bash
2+
# script.sh is a script to run Docker for testing. This is called by CI like
3+
# "bash .pfnci/script.sh py3.cpu". If a machine running the script has no
4+
# GPUs, this should fall back to CPU testing automatically. This script
5+
# requires that a corresponding Docker image is accessible from the machine.
6+
# TODO(imos): Enable external contributors to test this script on their
7+
# machines. Specifically, locate a Dockerfile generating chainer-ci-prep.*.
8+
#
9+
# Usage: .pfnci/script.sh [target]
10+
# - target is a test target (e.g., "py3.cpu").
11+
#
12+
# Environment variables:
13+
# - GPU (default: 0) ... Set a number of GPUs to GPU. GPU=0 disables GPU
14+
# testing.
15+
# - DRYRUN ... Set DRYRUN=1 for local testing. This disables destructive
16+
# actions and make the script print commands.
17+
# - XPYTEST ... Set XPYTEST=/path/to/xpytest-linux for testing xpytest. It will
18+
# replace xpytest installed inside a Docker image with the given binary.
19+
# It should be useful to test xpytest.
20+
21+
set -eu
22+
23+
cd "$(dirname "${BASH_SOURCE}")"/..
24+
25+
################################################################################
26+
# Main function
27+
################################################################################
28+
main() {
29+
TARGET="$1"
30+
31+
# Initialization.
32+
prepare_docker &
33+
prepare_xpytest &
34+
wait
35+
36+
# Prepare docker args.
37+
docker_args=(docker run --rm --volume="$(pwd):/src:ro")
38+
if [ "${GPU:-0}" != '0' ]; then
39+
docker_args+=(--ipc=host --privileged --env="GPU=${GPU}" --runtime=nvidia)
40+
fi
41+
if [ "${XPYTEST:-}" == '' ]; then
42+
docker_args+=(--volume="$(pwd)/bin/xpytest:/usr/local/bin/xpytest:ro")
43+
else
44+
docker_args+=(--volume="${XPYTEST}:/usr/local/bin/xpytest:ro")
45+
fi
46+
docker_args+=(--env="XPYTEST_NUM_THREADS=${XPYTEST_NUM_THREADS:-$(nproc)}")
47+
docker_args+=(--env="TEST_EXAMPLES=${TEST_EXAMPLES:-0}")
48+
49+
# Determine base image to use.
50+
docker_image=pytorch/pytorch:1.5.1-cuda10.1-cudnn7-runtime
51+
docker_args+=(--env="SLOW=${SLOW:-0}")
52+
53+
run "${docker_args[@]}" "${docker_image}" bash /src/.pfnci/run.sh "${TARGET}"
54+
}
55+
56+
################################################################################
57+
# Utility functions
58+
################################################################################
59+
60+
# run executes a command. If DRYRUN is enabled, run just prints the command.
61+
run() {
62+
echo '+' "$@"
63+
if [ "${DRYRUN:-}" == '' ]; then
64+
"$@"
65+
fi
66+
}
67+
68+
# prepare_docker makes docker use tmpfs to speed up.
69+
# CAVEAT: Do not use docker during this is running.
70+
prepare_docker() {
71+
# Mount tmpfs to docker's root directory to speed up.
72+
if [ "${CI:-}" != '' ]; then
73+
run service docker stop
74+
run mount -t tmpfs -o size=100% tmpfs /var/lib/docker
75+
run service docker start
76+
fi
77+
# Configure docker to pull images from gcr.io.
78+
run gcloud auth configure-docker
79+
}
80+
81+
# prepare_xpytest prepares xpytest.
82+
prepare_xpytest() {
83+
run mkdir -p bin
84+
run curl -L https://github.com/chainer/xpytest/releases/download/v0.1.0/xpytest-linux --output bin/xpytest
85+
run chmod +x bin/xpytest
86+
}
87+
88+
################################################################################
89+
# Bootstrap
90+
################################################################################
91+
main "$@"

0 commit comments

Comments
 (0)