Fix Python docs

apache · Aug 14, 2020 · 8ef1e88 · 8ef1e88
1 parent 86e96dc
commit 8ef1e88
Show file tree

Hide file tree

Showing 16 changed files with 37 additions and 198 deletions.
diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu
@@ -76,8 +76,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         # Python
         python3 \
         python3-pip \
-        python3-nose \
-        python3-nose-timer \
         ## Documentation
         doxygen \
         pandoc \

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_python b/ci/docker/Dockerfile.build.ubuntu_cpu_python
diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml
@@ -233,13 +233,6 @@ services:
       dockerfile: Dockerfile.build.ubuntu_cpu_jekyll
       cache_from:
         - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_jekyll:latest
-  ubuntu_cpu_python:
-    image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_python:latest
-    build:
-      context: .
-      dockerfile: Dockerfile.build.ubuntu_cpu_python
-      cache_from:
-        - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_python:latest
   ubuntu_blc:
     image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_blc:latest
     build:

diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -1128,11 +1128,6 @@ build_docs_setup() {
     mkdir -p $mxnetlib_folder
 }
 
-build_ubuntu_cpu_docs() {
-    build_ubuntu_cpu_openblas
-}
-
-
 build_jekyll_docs() {
     set -ex
 
@@ -1156,11 +1151,11 @@ build_python_docs() {
    build_docs_setup
 
    pushd docs/python_docs
-   eval "$(/work/miniconda/bin/conda shell.bash hook)"
-   conda env create -f environment.yml -p /work/conda_env
-   conda activate /work/conda_env
-   pip install themes/mx-theme
-   pip install -e /work/mxnet/python --user
+   python3 -m pip install -r requirements
+   python3 -m pip install themes/mx-theme
+   python3 -m pip install -e /work/mxnet/python --user
+
+   export PATH=/home/jenkins_slave/.local/bin:$PATH
 
    pushd python
    make clean

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
@@ -1044,22 +1044,6 @@ def test_qemu_armv8_cpu(lib_name) {
     }]
 }
 
-// This creates the MXNet binary needed for generating different docs sets
-def compile_unix_lite(lib_name) {
-    return ['MXNet lib': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/docs') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_docs', false)
-            utils.pack_lib(lib_name, mx_lib, false)
-          }
-        }
-      }
-    }]
-}
-
-
 def should_pack_website() {
   if (env.BRANCH_NAME) {
     if (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("new_")) {

diff --git a/ci/jenkins/Jenkinsfile_website_beta b/ci/jenkins/Jenkinsfile_website_beta
@@ -36,7 +36,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_ubuntu_cpu_openblas('libmxnet')
   ])
 
   utils.parallel_stage('Build Docs', [

diff --git a/ci/jenkins/Jenkinsfile_website_full b/ci/jenkins/Jenkinsfile_website_full
@@ -35,7 +35,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_unix_cpu_openblas('libmxnet')
   ])
 
   utils.parallel_stage('Build Docs', [

diff --git a/ci/jenkins/Jenkinsfile_website_full_pr b/ci/jenkins/Jenkinsfile_website_full_pr
@@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_ubuntu_cpu_openblas('libmxnet')
   ])
 
   utils.parallel_stage('Build Docs', [

diff --git a/ci/jenkins/Jenkinsfile_website_mxnet_build b/ci/jenkins/Jenkinsfile_website_mxnet_build
@@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_ubuntu_cpu_openblas('libmxnet')
   ])
 
 }

diff --git a/ci/jenkins/Jenkinsfile_website_nightly b/ci/jenkins/Jenkinsfile_website_nightly
@@ -35,7 +35,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_ubuntu_cpu_openblas('libmxnet')
   ])
 
   utils.parallel_stage('Build Docs', [

diff --git a/ci/jenkins/Jenkinsfile_website_python_docs b/ci/jenkins/Jenkinsfile_website_python_docs
@@ -34,7 +34,7 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_lite('libmxnet')
+    custom_steps.compile_ubuntu_cpu_openblas('libmxnet')
   ])
 
   utils.parallel_stage('Python Docs', [

diff --git a/docs/python_docs/README.md b/docs/python_docs/README.md
@@ -37,51 +37,16 @@ However, you may setup the website on macOS or Windows with or without a GPU.
 
 ### Prerequisites
 
-* [Conda >= 4.6.13](https://www.anaconda.com/distribution/#download-section) (install to PATH)
+To run the full build, including tests of all tutorials, **you will need at
+least two GPUs**. Distributed training is a key feature of MXNet, so multiple
+GPUs are required for running through some of the tutorials.
 
-You can update Conda with the following:
-
-```
-conda update conda
-```
-
-### GPU setup
-To run the full build, including tests of all tutorials,
-**you will need at least two GPUs**.
-Distributed training is a key feature of MXNet,
-so multiple GPUs are required for running through every tutorial.
-* [CUDA 9.2](https://developer.nvidia.com/cuda-downloads)
-
-### CPU-only setup
-In the `environment.yml` file:
-* Change `mxnet-cu92` to `mxnet`.
-
-### macOS setup
-In the `environment.yml` file:
-* Change `mxnet-cu92` to `mxnet`. (There is no CUDA package for mac anyway.)
-* Change `pytorch-cpu` to `pytorch`.
-* Change `torchvision-cpu` to `torchvision`.
-
-### Windows Setup
-If you have a GPU and have installed CUDA 9.2 you can leave the MXNet dependency alone.
-Otherwise, in the `environment.yml` file:
-* Change `mxnet-cu92` to `mxnet`.
-
-Install recommended software:
-* [git bash](https://gitforwindows.org/)
-* Be sure to install `Conda` in `PATH`
-* Install `make` from a `git bash` terminal with Admin rights
-    - [Install chocolatey](https://chocolatey.org/install)
-    - Use `choco to install make`
-* Restart terminals after installations to make sure PATH is set.
-    - The `choco`, `make`, and `conda` commands should work in `git bash`.
-
-### Conda environment setup
-Run the following commands from the project root (`python-docs`) to setup the environment.
+You need to install MXNet, for example, by following the build from source
+guide. Further, you need to install the Python requirements listed in the
+`requirements` file: 
 
 ```bash
-conda env create -f environment.yml
-source activate mxnet-docs
+python3 -m pip install -r requirements
 ```
 
 ## Build the docs
@@ -128,11 +93,3 @@ Optionally, one can run the following to launch the notedown plugin automaticall
 is not existing by run `jupyter notebook --generate-config`
 2. Add `c.NotebookApp.contents_manager_class = 'notedown.NotedownContentsManager'` to `~/.jupyter/jupyter_notebook_config.py`
 3. Simply run `jupyter notebook`
-
-## Troubleshooting
-Dependencies and the setup steps for this website are changing often. Here are some troubleshooting tips.
-
-* You might need to update the environment for the latest modules.
-```bash
-conda env update -f environment.yml
-```
diff --git a/docs/python_docs/python/Makefile b/docs/python_docs/python/Makefile
@@ -30,12 +30,12 @@ OBJ = $(patsubst %.rst, build/%.rst, $(RST)) \
 
 build/%.ipynb: %.md
 	@mkdir -p $(@D)
-	python scripts/md2ipynb.py $< $@
+	python3 scripts/md2ipynb.py $< $@
 
 
 build/%.rst: %.rst
 	@mkdir -p $(@D)
-	# python scripts/process_rst.py $< $@
+	# python3 scripts/process_rst.py $< $@
 	cp $< $@
 
 build/%: %

diff --git a/docs/python_docs/python/scripts/md2ipynb.py b/docs/python_docs/python/scripts/md2ipynb.py
@@ -39,7 +39,7 @@ def md2ipynb():
         print('%s: Evaluated %s in %f sec'%(src_fn, input_fn, time.time()-tic))
     # need to add language info to for syntax highlight
     notebook['metadata'].update({'language_info':{'name':'python'}})
-    with open(output_fn, 'w') as f:
+    with open(output_fn, 'w', encoding='utf-8') as f:
         f.write(nbformat.writes(notebook))
     print('%s: Write results into %s'%(src_fn, output_fn))
 

diff --git a/docs/python_docs/environment.yml → docs/python_docs/requirements b/docs/python_docs/environment.yml → docs/python_docs/requirements
@@ -14,25 +14,19 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-name: mxnet-docs
-channels:
-- conda-forge
-dependencies:
-- conda>=4.6.13
-- pip
-- python
-- jupyter
-- sphinx==2.4.0
-- matplotlib
-- notebook
-- pip:
-  - nbconvert==5.6.1
-  - nbsphinx==0.4.3
-  - recommonmark==0.6.0
-  - notedown==1.5.1
-  - pypandoc==1.4
-  - breathe==4.13.1
-  - mock==3.0.5
-  - awscli==1.16.266
-  - autodocsumm==0.1.12
+#
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+jupyter
+sphinx==2.4.0
+matplotlib
+notebook
+nbconvert==5.6.1
+nbsphinx==0.4.3
+recommonmark==0.6.0
+notedown==1.5.1
+pypandoc==1.4
+breathe==4.13.1
+mock==3.0.5
+awscli==1.16.266
+autodocsumm==0.1.12