From 8ef1e88b454c03cdaa2e45c7276dbb520b0f43a0 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 14 Aug 2020 18:11:11 +0000 Subject: [PATCH] Fix Python docs --- ci/docker/Dockerfile.build.ubuntu | 2 - ci/docker/Dockerfile.build.ubuntu_cpu_python | 46 --------------- ci/docker/docker-compose.yml | 7 --- ci/docker/install/ubuntu_docs.sh | 36 ------------ ci/docker/runtime_functions.sh | 15 ++--- ci/jenkins/Jenkins_steps.groovy | 16 ------ ci/jenkins/Jenkinsfile_website_beta | 2 +- ci/jenkins/Jenkinsfile_website_full | 2 +- ci/jenkins/Jenkinsfile_website_full_pr | 2 +- ci/jenkins/Jenkinsfile_website_mxnet_build | 2 +- ci/jenkins/Jenkinsfile_website_nightly | 2 +- ci/jenkins/Jenkinsfile_website_python_docs | 2 +- docs/python_docs/README.md | 57 +++---------------- docs/python_docs/python/Makefile | 4 +- docs/python_docs/python/scripts/md2ipynb.py | 2 +- .../{environment.yml => requirements} | 38 ++++++------- 16 files changed, 37 insertions(+), 198 deletions(-) delete mode 100644 ci/docker/Dockerfile.build.ubuntu_cpu_python delete mode 100755 ci/docker/install/ubuntu_docs.sh rename docs/python_docs/{environment.yml => requirements} (70%) diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu index abc3e8c6ce94..35119ae1335a 100644 --- a/ci/docker/Dockerfile.build.ubuntu +++ b/ci/docker/Dockerfile.build.ubuntu @@ -76,8 +76,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ # Python python3 \ python3-pip \ - python3-nose \ - python3-nose-timer \ ## Documentation doxygen \ pandoc \ diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_python b/ci/docker/Dockerfile.build.ubuntu_cpu_python deleted file mode 100644 index d1b0f6575100..000000000000 --- a/ci/docker/Dockerfile.build.ubuntu_cpu_python +++ /dev/null @@ -1,46 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU - -FROM ubuntu:16.04 - -WORKDIR /work/deps - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh - -COPY install/deb_ubuntu_ccache.sh /work/ -RUN /work/deb_ubuntu_ccache.sh - -COPY install/ubuntu_python.sh /work/ -COPY install/requirements /work/ -RUN /work/ubuntu_python.sh - -COPY install/ubuntu_docs.sh /work/ -RUN /work/ubuntu_docs.sh - -# Always last -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ - -WORKDIR /work/mxnet diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index 865abc128167..6d6b57048f30 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -233,13 +233,6 @@ services: dockerfile: Dockerfile.build.ubuntu_cpu_jekyll cache_from: - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_jekyll:latest - ubuntu_cpu_python: - image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_python:latest - build: - context: . - dockerfile: Dockerfile.build.ubuntu_cpu_python - cache_from: - - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_python:latest ubuntu_blc: image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_blc:latest build: diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh deleted file mode 100755 index 10a0016a4125..000000000000 --- a/ci/docker/install/ubuntu_docs.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -# Install dependencies -echo 'Installing dependencies...' -apt-get update || true -apt-get install -y \ - doxygen \ - pandoc - -# Can probably delete these and docs_requirements -wget -q https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh -chmod +x Miniconda2-latest-Linux-x86_64.sh -./Miniconda2-latest-Linux-x86_64.sh -b -p /work/miniconda - -echo 'Dependency installation complete.' diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1e988db11a18..c0c1cd28b472 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1128,11 +1128,6 @@ build_docs_setup() { mkdir -p $mxnetlib_folder } -build_ubuntu_cpu_docs() { - build_ubuntu_cpu_openblas -} - - build_jekyll_docs() { set -ex @@ -1156,11 +1151,11 @@ build_python_docs() { build_docs_setup pushd docs/python_docs - eval "$(/work/miniconda/bin/conda shell.bash hook)" - conda env create -f environment.yml -p /work/conda_env - conda activate /work/conda_env - pip install themes/mx-theme - pip install -e /work/mxnet/python --user + python3 -m pip install -r requirements + python3 -m pip install themes/mx-theme + python3 -m pip install -e /work/mxnet/python --user + + export PATH=/home/jenkins_slave/.local/bin:$PATH pushd python make clean diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index e2b0b04dea41..548811ab2099 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1044,22 +1044,6 @@ def test_qemu_armv8_cpu(lib_name) { }] } -// This creates the MXNet binary needed for generating different docs sets -def compile_unix_lite(lib_name) { - return ['MXNet lib': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_docs', false) - utils.pack_lib(lib_name, mx_lib, false) - } - } - } - }] -} - - def should_pack_website() { if (env.BRANCH_NAME) { if (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("new_")) { diff --git a/ci/jenkins/Jenkinsfile_website_beta b/ci/jenkins/Jenkinsfile_website_beta index a28e4b939784..1021e012e04d 100644 --- a/ci/jenkins/Jenkinsfile_website_beta +++ b/ci/jenkins/Jenkinsfile_website_beta @@ -36,7 +36,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_ubuntu_cpu_openblas('libmxnet') ]) utils.parallel_stage('Build Docs', [ diff --git a/ci/jenkins/Jenkinsfile_website_full b/ci/jenkins/Jenkinsfile_website_full index ca05c822b55e..03b576d47cb3 100644 --- a/ci/jenkins/Jenkinsfile_website_full +++ b/ci/jenkins/Jenkinsfile_website_full @@ -35,7 +35,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_unix_cpu_openblas('libmxnet') ]) utils.parallel_stage('Build Docs', [ diff --git a/ci/jenkins/Jenkinsfile_website_full_pr b/ci/jenkins/Jenkinsfile_website_full_pr index 7b78f9d190cd..df95d0ad28c1 100644 --- a/ci/jenkins/Jenkinsfile_website_full_pr +++ b/ci/jenkins/Jenkinsfile_website_full_pr @@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_ubuntu_cpu_openblas('libmxnet') ]) utils.parallel_stage('Build Docs', [ diff --git a/ci/jenkins/Jenkinsfile_website_mxnet_build b/ci/jenkins/Jenkinsfile_website_mxnet_build index 7be106cbb13b..1e95a4b1e046 100644 --- a/ci/jenkins/Jenkinsfile_website_mxnet_build +++ b/ci/jenkins/Jenkinsfile_website_mxnet_build @@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_ubuntu_cpu_openblas('libmxnet') ]) } diff --git a/ci/jenkins/Jenkinsfile_website_nightly b/ci/jenkins/Jenkinsfile_website_nightly index 39f7c48a924b..351527c8cc99 100644 --- a/ci/jenkins/Jenkinsfile_website_nightly +++ b/ci/jenkins/Jenkinsfile_website_nightly @@ -35,7 +35,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_ubuntu_cpu_openblas('libmxnet') ]) utils.parallel_stage('Build Docs', [ diff --git a/ci/jenkins/Jenkinsfile_website_python_docs b/ci/jenkins/Jenkinsfile_website_python_docs index 40367f9cab75..99e6d473faae 100644 --- a/ci/jenkins/Jenkinsfile_website_python_docs +++ b/ci/jenkins/Jenkinsfile_website_python_docs @@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite('libmxnet') + custom_steps.compile_ubuntu_cpu_openblas('libmxnet') ]) utils.parallel_stage('Python Docs', [ diff --git a/docs/python_docs/README.md b/docs/python_docs/README.md index a547e70d16d3..68f250e50e6b 100644 --- a/docs/python_docs/README.md +++ b/docs/python_docs/README.md @@ -37,51 +37,16 @@ However, you may setup the website on macOS or Windows with or without a GPU. ### Prerequisites -* [Conda >= 4.6.13](https://www.anaconda.com/distribution/#download-section) (install to PATH) +To run the full build, including tests of all tutorials, **you will need at +least two GPUs**. Distributed training is a key feature of MXNet, so multiple +GPUs are required for running through some of the tutorials. -You can update Conda with the following: - -``` -conda update conda -``` - -### GPU setup -To run the full build, including tests of all tutorials, -**you will need at least two GPUs**. -Distributed training is a key feature of MXNet, -so multiple GPUs are required for running through every tutorial. -* [CUDA 9.2](https://developer.nvidia.com/cuda-downloads) - -### CPU-only setup -In the `environment.yml` file: -* Change `mxnet-cu92` to `mxnet`. - -### macOS setup -In the `environment.yml` file: -* Change `mxnet-cu92` to `mxnet`. (There is no CUDA package for mac anyway.) -* Change `pytorch-cpu` to `pytorch`. -* Change `torchvision-cpu` to `torchvision`. - -### Windows Setup -If you have a GPU and have installed CUDA 9.2 you can leave the MXNet dependency alone. -Otherwise, in the `environment.yml` file: -* Change `mxnet-cu92` to `mxnet`. - -Install recommended software: -* [git bash](https://gitforwindows.org/) -* Be sure to install `Conda` in `PATH` -* Install `make` from a `git bash` terminal with Admin rights - - [Install chocolatey](https://chocolatey.org/install) - - Use `choco to install make` -* Restart terminals after installations to make sure PATH is set. - - The `choco`, `make`, and `conda` commands should work in `git bash`. - -### Conda environment setup -Run the following commands from the project root (`python-docs`) to setup the environment. +You need to install MXNet, for example, by following the build from source +guide. Further, you need to install the Python requirements listed in the +`requirements` file: ```bash -conda env create -f environment.yml -source activate mxnet-docs +python3 -m pip install -r requirements ``` ## Build the docs @@ -128,11 +93,3 @@ Optionally, one can run the following to launch the notedown plugin automaticall is not existing by run `jupyter notebook --generate-config` 2. Add `c.NotebookApp.contents_manager_class = 'notedown.NotedownContentsManager'` to `~/.jupyter/jupyter_notebook_config.py` 3. Simply run `jupyter notebook` - -## Troubleshooting -Dependencies and the setup steps for this website are changing often. Here are some troubleshooting tips. - -* You might need to update the environment for the latest modules. -```bash -conda env update -f environment.yml -``` diff --git a/docs/python_docs/python/Makefile b/docs/python_docs/python/Makefile index 0c5450887a16..be811221458c 100644 --- a/docs/python_docs/python/Makefile +++ b/docs/python_docs/python/Makefile @@ -30,12 +30,12 @@ OBJ = $(patsubst %.rst, build/%.rst, $(RST)) \ build/%.ipynb: %.md @mkdir -p $(@D) - python scripts/md2ipynb.py $< $@ + python3 scripts/md2ipynb.py $< $@ build/%.rst: %.rst @mkdir -p $(@D) - # python scripts/process_rst.py $< $@ + # python3 scripts/process_rst.py $< $@ cp $< $@ build/%: % diff --git a/docs/python_docs/python/scripts/md2ipynb.py b/docs/python_docs/python/scripts/md2ipynb.py index 7793dea39847..9d1c2c173330 100644 --- a/docs/python_docs/python/scripts/md2ipynb.py +++ b/docs/python_docs/python/scripts/md2ipynb.py @@ -39,7 +39,7 @@ def md2ipynb(): print('%s: Evaluated %s in %f sec'%(src_fn, input_fn, time.time()-tic)) # need to add language info to for syntax highlight notebook['metadata'].update({'language_info':{'name':'python'}}) - with open(output_fn, 'w') as f: + with open(output_fn, 'w', encoding='utf-8') as f: f.write(nbformat.writes(notebook)) print('%s: Write results into %s'%(src_fn, output_fn)) diff --git a/docs/python_docs/environment.yml b/docs/python_docs/requirements similarity index 70% rename from docs/python_docs/environment.yml rename to docs/python_docs/requirements index 7856a076889b..2556cc3c72a1 100644 --- a/docs/python_docs/environment.yml +++ b/docs/python_docs/requirements @@ -14,25 +14,19 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -name: mxnet-docs -channels: -- conda-forge -dependencies: -- conda>=4.6.13 -- pip -- python -- jupyter -- sphinx==2.4.0 -- matplotlib -- notebook -- pip: - - nbconvert==5.6.1 - - nbsphinx==0.4.3 - - recommonmark==0.6.0 - - notedown==1.5.1 - - pypandoc==1.4 - - breathe==4.13.1 - - mock==3.0.5 - - awscli==1.16.266 - - autodocsumm==0.1.12 +# +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image +jupyter +sphinx==2.4.0 +matplotlib +notebook +nbconvert==5.6.1 +nbsphinx==0.4.3 +recommonmark==0.6.0 +notedown==1.5.1 +pypandoc==1.4 +breathe==4.13.1 +mock==3.0.5 +awscli==1.16.266 +autodocsumm==0.1.12 \ No newline at end of file