From 78109615ca04419fb81ec07271ddee8b9d5382eb Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Thu, 28 Jan 2021 19:17:09 +0100 Subject: [PATCH] Remove unmaintained BLC (#19801) --- ci/docker/Dockerfile.build.ubuntu_blc | 41 ------- ci/docker/docker-compose.yml | 7 -- ci/docker/install/ubuntu_core.sh | 70 ------------ ci/docker/install/ubuntu_npm_blc.sh | 37 ------ ci/docker/install/ubuntu_python.sh | 32 ------ ci/docker/install/ubuntu_tutorials.sh | 28 ----- ci/docker/runtime_functions.sh | 10 -- .../JenkinsfileForBLC | 59 ---------- .../broken_link_checker_test/README.md | 27 ----- .../broken_link_checker.sh | 30 ----- .../test_broken_links.py | 105 ------------------ 11 files changed, 446 deletions(-) delete mode 100644 ci/docker/Dockerfile.build.ubuntu_blc delete mode 100755 ci/docker/install/ubuntu_core.sh delete mode 100755 ci/docker/install/ubuntu_npm_blc.sh delete mode 100755 ci/docker/install/ubuntu_python.sh delete mode 100755 ci/docker/install/ubuntu_tutorials.sh delete mode 100755 tests/nightly/broken_link_checker_test/JenkinsfileForBLC delete mode 100644 tests/nightly/broken_link_checker_test/README.md delete mode 100755 tests/nightly/broken_link_checker_test/broken_link_checker.sh delete mode 100644 tests/nightly/broken_link_checker_test/test_broken_links.py diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc deleted file mode 100644 index a413648366b2..000000000000 --- a/ci/docker/Dockerfile.build.ubuntu_blc +++ /dev/null @@ -1,41 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU - -FROM ubuntu:16.04 - -WORKDIR /work/deps - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_python.sh /work/ -COPY install/requirements /work/ -RUN /work/ubuntu_python.sh -COPY install/ubuntu_npm_blc.sh /work/ -RUN /work/ubuntu_npm_blc.sh - -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ - -WORKDIR /work/mxnet -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index 7c0ef4382087..aa4cd06bb852 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -213,10 +213,3 @@ services: dockerfile: Dockerfile.build.ubuntu_cpu_jekyll cache_from: - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_jekyll:latest - ubuntu_blc: - image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_blc:latest - build: - context: . - dockerfile: Dockerfile.build.ubuntu_blc - cache_from: - - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_blc:latest diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh deleted file mode 100755 index 9ff3ac70b859..000000000000 --- a/ci/docker/install/ubuntu_core.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -# FIXME(larroy) enable in a different PR -#perl -pi -e 's/archive.ubuntu.com/us-west-2.ec2.archive.ubuntu.com/' /etc/apt/sources.list -apt-get update || true - -# Avoid interactive package installers such as tzdata. -export DEBIAN_FRONTEND=noninteractive - -apt-get install -y \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - git \ - libatlas-base-dev \ - libcurl4-openssl-dev \ - libjemalloc-dev \ - libhdf5-dev \ - liblapack-dev \ - libopenblas-dev \ - libopencv-dev \ - libturbojpeg \ - libzmq3-dev \ - libtinfo-dev \ - zlib1g-dev \ - libedit-dev \ - libxml2-dev \ - libprotobuf-dev \ - protobuf-compiler \ - ninja-build \ - software-properties-common \ - sudo \ - unzip \ - vim-nox \ - wget - -# Use libturbojpeg package as it is correctly compiled with -fPIC flag -# https://github.com/HaxeFoundation/hashlink/issues/147 -ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so - - -# CMake 3.13.2+ is required -mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh -sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license -ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.13.5-Linux-x86_64.sh -cmake --version diff --git a/ci/docker/install/ubuntu_npm_blc.sh b/ci/docker/install/ubuntu_npm_blc.sh deleted file mode 100755 index 59caa9f88bf4..000000000000 --- a/ci/docker/install/ubuntu_npm_blc.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -echo 'Installing npm...' -apt-get update || true -apt-get install -y npm - -echo "Obtaining NodeJS version 8.x" -curl -sL https://deb.nodesource.com/setup_8.x | bash - - -echo "Installing nodejs" -apt-get install -y nodejs - -# Install broken link checker utility -echo "Installing broken link checker utility" -npm install broken-link-checker -g - diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh deleted file mode 100755 index 6234aac12283..000000000000 --- a/ci/docker/install/ubuntu_python.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -# install libraries for mxnet's python package on ubuntu -apt-get update || true -apt-get install -y python-dev python3-dev virtualenv wget -apt-get install -y libprotobuf-dev protobuf-compiler - -# the version of the pip shipped with ubuntu may be too lower, install a recent version here -wget -nv https://bootstrap.pypa.io/get-pip.py -python3 get-pip.py -pip3 install -r /work/requirements diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/docker/install/ubuntu_tutorials.sh deleted file mode 100755 index 469df6190ea4..000000000000 --- a/ci/docker/install/ubuntu_tutorials.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -apt-get update || true -apt-get install graphviz python-opencv - -# sckit-learn past version 0.20 does not support python version 2 and 3.4 -pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz==0.8.4 tqdm mxboard scipy gluoncv diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4f5276bcd4b3..c95455c784af 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -690,10 +690,6 @@ build_ubuntu_gpu_large_tensor() { ninja } -build_ubuntu_blc() { - echo "pass" -} - # Testing sanity_check() { @@ -1345,12 +1341,6 @@ build_static_python_cu102() { popd } -# broken_link_checker -broken_link_checker() { - set -ex - ./tests/nightly/broken_link_checker_test/broken_link_checker.sh -} - # artifact repository unit tests test_artifact_repository() { set -ex diff --git a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC deleted file mode 100755 index 8a96a4f5013c..000000000000 --- a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC +++ /dev/null @@ -1,59 +0,0 @@ -// -*- mode: groovy -*- -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -//This is a Jenkinsfile for the broken link checker test. - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - stage('BLC'){ - parallel 'BrokenLinkChecker: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/brokenLinkChecker') { - timeout(time: 60, unit: 'MINUTES') { - try { - utils.init_git() - utils.docker_run('ubuntu_blc', 'broken_link_checker', false) - } catch(Exception ex) { - sh "echo Broken Link Checker failed with exception - " + ex.toString() - } - } - } - } - } - } -} -, -failure_handler: -{ - if (currentBuild.result == "FAILURE") { - emailext body: '''https://mxnet.apache.org broken link test summary: - | - |Please view the logs at ${BUILD_URL} - | - |${BUILD_LOG_EXCERPT, start="START - Broken links summary", end="END - Broken links summary"}''', - replyTo: '${EMAIL}', subject: '[BROKEN LINK CHECKER FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/tests/nightly/broken_link_checker_test/README.md b/tests/nightly/broken_link_checker_test/README.md deleted file mode 100644 index d493390c844f..000000000000 --- a/tests/nightly/broken_link_checker_test/README.md +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - -# Broken link checker test - -This folder contains the scripts that are required to run the nightly job of checking the broken links. - -## JenkinsfileForBLC -This is configuration file for jenkins job. - -## Details -The `broken_link_checker.sh` is a top level script that invokes the `test_broken_links.py` and `check_regression.sh` scripts. -The `test_broken_links.py` invokes broken link checker tool (blc) from nodeJs and reports the list of URLs that are not accessible. diff --git a/tests/nightly/broken_link_checker_test/broken_link_checker.sh b/tests/nightly/broken_link_checker_test/broken_link_checker.sh deleted file mode 100755 index 450cd65b8d96..000000000000 --- a/tests/nightly/broken_link_checker_test/broken_link_checker.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#Author: Amol Lele - -#software-properties-common, curl, npm are installed in the docker container 'ubuntu_blc' - -echo "Invoking broken_link_checker.sh script" -echo `pwd` -cd tests/nightly/broken_link_checker_test -echo `pwd` - -echo "Running test_broken_links.py" -python test_broken_links.py diff --git a/tests/nightly/broken_link_checker_test/test_broken_links.py b/tests/nightly/broken_link_checker_test/test_broken_links.py deleted file mode 100644 index 2c0634f9522a..000000000000 --- a/tests/nightly/broken_link_checker_test/test_broken_links.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import sys -from subprocess import check_output, STDOUT, CalledProcessError - - -def prepare_link_test_result(command_output): - # Constant string patterns - NEW_PAGE_TEST_START_REGEX = "Getting links from:" - BROKEN_PAGE_START_REGEX = "BROKEN" - PAGE_TEST_END_REGEX = "Finished! " - - # Whitelisted broken links patterns - HTTP_403_REGEX = "(HTTP_403)" - HTTP_401_REGEX = "(HTTP_401)" - HTTP_409_REGEX = "(HTTP_409)" - HTTP_3XX_REGEX = "(HTTP_3" - BLC_UNKNOWN_REGEX = "(BLC_UNKNOWN)" - HTTP_UNDEFINED = "HTTP_undefined" - FALSE_SCALA_API_DOC_LINK = "java$lang.html" - FALSE_SCALA_API_DEPRECATED_LINK = "api/scala/docs/deprecated-list.html" - FALSE_PAPER_LINK = "https://static.googleusercontent.com/media/research.google.com/en/pubs/archive/45488.pdf" - - # Initialize flags with happy case - current_page = "" - current_page_broken = False - current_page_broken_links = "" - - broken_links_count = 0 - broken_links_summary = "" - - for line in command_output.splitlines(): - if line.startswith(NEW_PAGE_TEST_START_REGEX): - # New page test is starting. Reset the flags - current_page = line.split(NEW_PAGE_TEST_START_REGEX)[1] - current_page_broken = False - current_page_broken_links = "" - - if line.find(BROKEN_PAGE_START_REGEX) != -1: - # Skip (401, 403, 409, unknown issues) - if HTTP_403_REGEX not in line and HTTP_401_REGEX not in line and HTTP_409_REGEX not in line and HTTP_3XX_REGEX not in line and BLC_UNKNOWN_REGEX not in line and HTTP_UNDEFINED not in line and FALSE_SCALA_API_DOC_LINK not in line and FALSE_SCALA_API_DEPRECATED_LINK not in line and FALSE_PAPER_LINK not in line: - current_page_broken = True - current_page_broken_links += line.split(BROKEN_PAGE_START_REGEX)[1] + "\n" - - if line.startswith(PAGE_TEST_END_REGEX): - if current_page_broken: - broken_links_count += 1 - broken_links_summary += "\nURL - " + current_page - broken_links_summary += "\nBroken Links\n" + current_page_broken_links - - return broken_links_count, broken_links_summary - -# Command to check broken links -# Reference - https://www.npmjs.com/package/broken-link-checker -cmd = "blc https://mxnet.apache.org -ro" -broken_links_count = 0 -broken_links_summary = "" - -text_file = open("./blc_output.txt", 'w') -command_output = "" -print("Starting broken link test with command $ " + cmd) -try: - command_output = check_output(cmd, stderr=STDOUT, shell=True) -except CalledProcessError as ex: - if ex.returncode > 1: - print("Failed to do broken link test. Console output : \n" + ex.output) - sys.exit(ex.returncode) - command_output = ex.output - -text_file.write(command_output) -text_file.close() -broken_links_count, broken_links_summary = prepare_link_test_result(command_output) -# These START and END string in output is used to parse the script output in automated scripts and nightly jobs. -print("START - Broken links count") -print(broken_links_count) -print("END - Broken links count") - -print("START - Broken links summary") - -if broken_links_count == 0: - print("No broken links in https://mxnet.apache.org") - print("END - Broken links summary") -else: - print(broken_links_summary) - print("END - Broken links summary") - # Fail the job as we found the broken links - sys.exit(-1)