From 65819ccd8f134013a4ee21d5ccc94cdb78b3472b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 27 Oct 2020 06:48:32 +0000 Subject: [PATCH 1/7] initial commit --- CMakeLists.txt | 5 ++ .../lib_external_ops/CMakeLists.txt | 27 ++++++++++ example/extensions/lib_external_ops/README.md | 53 +++++++++++++++++++ .../extensions/lib_external_ops/init_lib.cc | 14 +++++ .../extensions/lib_external_ops/min_ex-inl.h | 41 ++++++++++++++ example/extensions/lib_external_ops/min_ex.cc | 15 ++++++ example/extensions/lib_external_ops/min_ex.cu | 10 ++++ .../lib_external_ops/test_loading.py | 17 ++++++ 8 files changed, 182 insertions(+) create mode 100644 example/extensions/lib_external_ops/CMakeLists.txt create mode 100644 example/extensions/lib_external_ops/README.md create mode 100644 example/extensions/lib_external_ops/init_lib.cc create mode 100644 example/extensions/lib_external_ops/min_ex-inl.h create mode 100644 example/extensions/lib_external_ops/min_ex.cc create mode 100644 example/extensions/lib_external_ops/min_ex.cu create mode 100644 example/extensions/lib_external_ops/test_loading.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 07075d7523a4..196f8507f25a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ option(USE_TENSORRT "Enable inference optimization with TensorRT." OFF) option(USE_ASAN "Enable Clang/GCC ASAN sanitizers." OFF) cmake_dependent_option(ENABLE_TESTCOVERAGE "Enable compilation with test coverage metric output" OFF "NOT MSVC" OFF) option(USE_INT64_TENSOR_SIZE "Use int64_t to represent the total number of elements in a tensor" OFF) +option(BUILD_EXTENSION "Path to extension to build" "") option(BUILD_CYTHON_MODULES "Build cython modules." OFF) option(LOG_FATAL_THROW "Log exceptions but do not abort" ON) cmake_dependent_option(USE_SPLIT_ARCH_DLL "Build a separate DLL for each Cuda arch (Windows only)." ON "MSVC" OFF) @@ -787,6 +788,10 @@ add_library(transposerowsp_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extens add_library(subgraph_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_subgraph/subgraph_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc) add_library(pass_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_pass/pass_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc) +if(IS_DIRECTORY ${BUILD_EXTENSION}) + add_subdirectory(${BUILD_EXTENSION} ${BUILD_EXTENSION}/build) +endif() + target_include_directories(customop_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet) target_include_directories(transposecsr_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet) target_include_directories(transposerowsp_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet) diff --git a/example/extensions/lib_external_ops/CMakeLists.txt b/example/extensions/lib_external_ops/CMakeLists.txt new file mode 100644 index 000000000000..cebf8b79c4b8 --- /dev/null +++ b/example/extensions/lib_external_ops/CMakeLists.txt @@ -0,0 +1,27 @@ +# specify CXX sources +FILE(GLOB CXX_SRCS + # Required files + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/lib_api.cc + # Your custom files + ${CMAKE_CURRENT_SOURCE_DIR}/init_lib.cc + ${CMAKE_CURRENT_SOURCE_DIR}/min_ex.cc + ) + +# create library & set libraries +add_library(external_lib SHARED ${CXX_SRCS}) +target_link_libraries(external_lib PUBLIC mxnet) + +# generic GPU stuff +if(USE_CUDA) + # specify GPU sources (optional) + FILE(GLOB CU_SRCS "*.cu") + target_sources(external_lib PUBLIC ${CU_SRCS}) +endif(USE_CUDA) + +if(UNIX) + # unix-specific stuff + if(USE_CUDA) + # unix+GPU-specific stuff + target_compile_options(external_lib PUBLIC -shared) + endif(USE_CUDA) +endif() diff --git a/example/extensions/lib_external_ops/README.md b/example/extensions/lib_external_ops/README.md new file mode 100644 index 000000000000..218b9c891d17 --- /dev/null +++ b/example/extensions/lib_external_ops/README.md @@ -0,0 +1,53 @@ +External Operators Example and Tutorial +======================================= + +## Introduction + +Extending MXNet with custom components used to mean distributing a custom fork. This feature allows adding custom components to MXNet by dynamically loading external libraries at runtime. Currently it is only supported on Linux systems (Windows and Mac are __NOT__ supported). + +## Getting Started + +### Have MXNet Ready + +For this tutorial, clone MXNet from source like: +``` +git clone https://github.com/apache/incubator-mxnet.git --recursive --init +``` + +Build MXNet like: +``` +cp config/linux.cmake config.cmake +mkdir build +cd build +cmake .. +cmake --build . +``` + +## Run An Example + +This example shows compiling a custom backend operator and then dynamically loading it into MXNet at runtime. Go to the **lib_external_ops** directory and follow these steps: + +1. Touch or modify the **min_ex.cc** and/or **min_ex-inl.h** file(s) +2. Go into the **build** directory that was created when building MXNet. +3. Run `cmake .. -DBUILD_EXTENSION=$(pwd)/../example/extensions/lib_external_ops` +4. Run `cmake --build .` +5. Go to the **example/extensions/lib_external_ops** directory again +6. Run `python test_loading.py` to execute the test program. You should see the following output: +``` +Operator not registered yet +MXNet version 20000 supported +[] +Operator executed successfully +``` + +## Writing an External Operator Library +To build your own library containing custom components, compose a C++ source file like `mycomp_lib.cc`, include the `lib_api.h` header file, compile the `lib_api.cc` file, and implement the following required function: +- `initialize` - Library Initialization Function + +Then create a CMakeLists.txt file and set `mxnet` as a link library like: +``` +add_library(external_lib SHARED ${SRCS}) +target_link_libraries(external_lib PUBLIC mxnet) +``` + +Next, build MXNet and set the path to your directory with the CMakeLists.txt file via the `BUILD_EXTENSION` option. This will build your library with all of the MXNet includes. diff --git a/example/extensions/lib_external_ops/init_lib.cc b/example/extensions/lib_external_ops/init_lib.cc new file mode 100644 index 000000000000..7603bfacb1e0 --- /dev/null +++ b/example/extensions/lib_external_ops/init_lib.cc @@ -0,0 +1,14 @@ +#include +#include "mxnet/lib_api.h" + +using namespace mxnet::ext; + +MXReturnValue initialize(int version) { + if (version >= 10700) { + std::cout << "MXNet version " << version << " supported" << std::endl; + return MX_SUCCESS; + } else { + MX_ERROR_MSG << "MXNet version " << version << " not supported"; + return MX_FAIL; + } +} diff --git a/example/extensions/lib_external_ops/min_ex-inl.h b/example/extensions/lib_external_ops/min_ex-inl.h new file mode 100644 index 000000000000..de0ee2e9c67a --- /dev/null +++ b/example/extensions/lib_external_ops/min_ex-inl.h @@ -0,0 +1,41 @@ +#ifndef MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_ +#define MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_ + +#include +#include +#include +#include "operator/mxnet_op.h" +#include "operator/operator_common.h" +#include "operator/elemwise_op_common.h" + +namespace mxnet { +namespace op { + +template +void MinExForward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + //do nothing +} + + +inline bool MinExOpShape(const nnvm::NodeAttrs& attrs, + mxnet::ShapeVector* in_attrs, + mxnet::ShapeVector* out_attrs) { + //do nothing + return true; +} + +inline bool MinExOpType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + //do nothing + return true; +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_ diff --git a/example/extensions/lib_external_ops/min_ex.cc b/example/extensions/lib_external_ops/min_ex.cc new file mode 100644 index 000000000000..1c4381885852 --- /dev/null +++ b/example/extensions/lib_external_ops/min_ex.cc @@ -0,0 +1,15 @@ +#include "min_ex-inl.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(min_ex) +.describe("some description") +.set_num_inputs(0) +.set_num_outputs(0) +.set_attr("FInferShape", MinExOpShape) +.set_attr("FInferType", MinExOpType) +.set_attr("FCompute", MinExForward); + +} // namespace op +} // namespace mxnet diff --git a/example/extensions/lib_external_ops/min_ex.cu b/example/extensions/lib_external_ops/min_ex.cu new file mode 100644 index 000000000000..34c86b6b21d5 --- /dev/null +++ b/example/extensions/lib_external_ops/min_ex.cu @@ -0,0 +1,10 @@ +#include "./min_ex-inl.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(min_ex) +.set_attr("FCompute", MinExForward); + +} // namespace op +} // namespace mxnet diff --git a/example/extensions/lib_external_ops/test_loading.py b/example/extensions/lib_external_ops/test_loading.py new file mode 100644 index 000000000000..4679499967be --- /dev/null +++ b/example/extensions/lib_external_ops/test_loading.py @@ -0,0 +1,17 @@ +import mxnet as mx +import os + +# check if operator exists +if hasattr(mx.nd,'min_ex'): + raise Exception('Operator already loaded') +else: + print('Operator not registered yet') + +# test loading library +if (os.name=='posix'): + path = os.path.abspath('build/libexternal_lib.so') + mx.library.load(path,False) + +# execute operator +print(mx.nd.min_ex()) +print('Operator executed successfully') From 0616ad8210b28867751206eb3e6b706d6782fc0c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 27 Oct 2020 07:10:34 +0000 Subject: [PATCH 2/7] license fix --- example/extensions/lib_external_ops/README.md | 17 +++++++++++++ .../extensions/lib_external_ops/init_lib.cc | 25 +++++++++++++++++++ .../extensions/lib_external_ops/min_ex-inl.h | 25 +++++++++++++++++++ example/extensions/lib_external_ops/min_ex.cc | 25 +++++++++++++++++++ example/extensions/lib_external_ops/min_ex.cu | 25 +++++++++++++++++++ .../lib_external_ops/test_loading.py | 25 +++++++++++++++++++ 6 files changed, 142 insertions(+) diff --git a/example/extensions/lib_external_ops/README.md b/example/extensions/lib_external_ops/README.md index 218b9c891d17..a484b07c4bdf 100644 --- a/example/extensions/lib_external_ops/README.md +++ b/example/extensions/lib_external_ops/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + External Operators Example and Tutorial ======================================= diff --git a/example/extensions/lib_external_ops/init_lib.cc b/example/extensions/lib_external_ops/init_lib.cc index 7603bfacb1e0..efc5eb7a0266 100644 --- a/example/extensions/lib_external_ops/init_lib.cc +++ b/example/extensions/lib_external_ops/init_lib.cc @@ -1,3 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2020 by Contributors + * \file init_lib.cc + * \brief initialize function implementation library file + */ + #include #include "mxnet/lib_api.h" diff --git a/example/extensions/lib_external_ops/min_ex-inl.h b/example/extensions/lib_external_ops/min_ex-inl.h index de0ee2e9c67a..79ce5d407890 100644 --- a/example/extensions/lib_external_ops/min_ex-inl.h +++ b/example/extensions/lib_external_ops/min_ex-inl.h @@ -1,3 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2020 by Contributors + * \file min_ex-inl.h + * \brief example external operator header file + */ + #ifndef MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_ #define MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_ diff --git a/example/extensions/lib_external_ops/min_ex.cc b/example/extensions/lib_external_ops/min_ex.cc index 1c4381885852..cb9f6dda8b1e 100644 --- a/example/extensions/lib_external_ops/min_ex.cc +++ b/example/extensions/lib_external_ops/min_ex.cc @@ -1,3 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2020 by Contributors + * \file min_ex.cc + * \brief example external operator source file + */ + #include "min_ex-inl.h" namespace mxnet { diff --git a/example/extensions/lib_external_ops/min_ex.cu b/example/extensions/lib_external_ops/min_ex.cu index 34c86b6b21d5..6257ea703ba3 100644 --- a/example/extensions/lib_external_ops/min_ex.cu +++ b/example/extensions/lib_external_ops/min_ex.cu @@ -1,3 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2020 by Contributors + * \file min_ex.cu + * \brief example external operator CUDA source file + */ + #include "./min_ex-inl.h" namespace mxnet { diff --git a/example/extensions/lib_external_ops/test_loading.py b/example/extensions/lib_external_ops/test_loading.py index 4679499967be..381bfa2a0ed9 100644 --- a/example/extensions/lib_external_ops/test_loading.py +++ b/example/extensions/lib_external_ops/test_loading.py @@ -1,3 +1,28 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=arguments-differ + +# This test checks if dynamic loading of library into MXNet is successful +# and checks the computation of an external operator + import mxnet as mx import os From 8b0663dc6071d0f2a4992c4a0e0df1582a1bca1c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 31 Oct 2020 05:23:11 +0000 Subject: [PATCH 3/7] changed path var, formatting --- CMakeLists.txt | 6 +++--- example/extensions/lib_external_ops/README.md | 4 ++-- example/extensions/lib_external_ops/test_loading.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 196f8507f25a..9ffda36f3a4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,7 @@ option(USE_TENSORRT "Enable inference optimization with TensorRT." OFF) option(USE_ASAN "Enable Clang/GCC ASAN sanitizers." OFF) cmake_dependent_option(ENABLE_TESTCOVERAGE "Enable compilation with test coverage metric output" OFF "NOT MSVC" OFF) option(USE_INT64_TENSOR_SIZE "Use int64_t to represent the total number of elements in a tensor" OFF) -option(BUILD_EXTENSION "Path to extension to build" "") +option(BUILD_EXTENSION_PATH "Path to extension to build" "") option(BUILD_CYTHON_MODULES "Build cython modules." OFF) option(LOG_FATAL_THROW "Log exceptions but do not abort" ON) cmake_dependent_option(USE_SPLIT_ARCH_DLL "Build a separate DLL for each Cuda arch (Windows only)." ON "MSVC" OFF) @@ -788,8 +788,8 @@ add_library(transposerowsp_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extens add_library(subgraph_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_subgraph/subgraph_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc) add_library(pass_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_pass/pass_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc) -if(IS_DIRECTORY ${BUILD_EXTENSION}) - add_subdirectory(${BUILD_EXTENSION} ${BUILD_EXTENSION}/build) +if(IS_DIRECTORY ${BUILD_EXTENSION_PATH}) + add_subdirectory(${BUILD_EXTENSION_PATH} ${BUILD_EXTENSION_PATH}/build) endif() target_include_directories(customop_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet) diff --git a/example/extensions/lib_external_ops/README.md b/example/extensions/lib_external_ops/README.md index a484b07c4bdf..40291e13dfa9 100644 --- a/example/extensions/lib_external_ops/README.md +++ b/example/extensions/lib_external_ops/README.md @@ -46,7 +46,7 @@ This example shows compiling a custom backend operator and then dynamically load 1. Touch or modify the **min_ex.cc** and/or **min_ex-inl.h** file(s) 2. Go into the **build** directory that was created when building MXNet. -3. Run `cmake .. -DBUILD_EXTENSION=$(pwd)/../example/extensions/lib_external_ops` +3. Run `cmake .. -DBUILD_EXTENSION_PATH=$(pwd)/../example/extensions/lib_external_ops` 4. Run `cmake --build .` 5. Go to the **example/extensions/lib_external_ops** directory again 6. Run `python test_loading.py` to execute the test program. You should see the following output: @@ -67,4 +67,4 @@ add_library(external_lib SHARED ${SRCS}) target_link_libraries(external_lib PUBLIC mxnet) ``` -Next, build MXNet and set the path to your directory with the CMakeLists.txt file via the `BUILD_EXTENSION` option. This will build your library with all of the MXNet includes. +Next, build MXNet and set the path to your directory with the CMakeLists.txt file via the `BUILD_EXTENSION_PATH` option. This will build your library with all of the MXNet includes. diff --git a/example/extensions/lib_external_ops/test_loading.py b/example/extensions/lib_external_ops/test_loading.py index 381bfa2a0ed9..1b4357c7382e 100644 --- a/example/extensions/lib_external_ops/test_loading.py +++ b/example/extensions/lib_external_ops/test_loading.py @@ -27,15 +27,15 @@ import os # check if operator exists -if hasattr(mx.nd,'min_ex'): +if hasattr(mx.nd, 'min_ex'): raise Exception('Operator already loaded') else: print('Operator not registered yet') # test loading library -if (os.name=='posix'): +if (os.name == 'posix'): path = os.path.abspath('build/libexternal_lib.so') - mx.library.load(path,False) + mx.library.load(path, False) # execute operator print(mx.nd.min_ex()) From 0e31d9227c8faa8a88449a9c81c3c7b283ad9ef9 Mon Sep 17 00:00:00 2001 From: Manu Seth Date: Fri, 6 Nov 2020 07:51:47 +0000 Subject: [PATCH 4/7] add test to linux stages in ci --- ci/docker/runtime_functions.sh | 10 ++++++++++ ci/jenkins/Jenkins_steps.groovy | 10 +++++----- tests/python/gpu/test_extensions_gpu.py | 25 ++++++++++++++++++++++-- tests/python/unittest/test_extensions.py | 25 ++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 40405b96163e..8a34e734eee1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -268,6 +268,7 @@ build_centos7_cpu() { -DUSE_MKLDNN=OFF \ -DUSE_DIST_KVSTORE=ON \ -DUSE_CUDA=OFF \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -299,6 +300,7 @@ build_centos7_gpu() { -DUSE_CUDA=ON \ -DMXNET_CUDA_ARCH="$CI_CMAKE_CUDA_ARCH" \ -DUSE_DIST_KVSTORE=ON\ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -320,6 +322,7 @@ build_ubuntu_cpu_openblas() { -DUSE_CUDA=OFF \ -DUSE_DIST_KVSTORE=ON \ -DBUILD_CYTHON_MODULES=ON \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -335,6 +338,7 @@ build_ubuntu_cpu_mkl() { -DUSE_TVM_OP=ON \ -DUSE_MKL_IF_AVAILABLE=ON \ -DUSE_BLAS=MKL \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -GNinja /work/mxnet ninja } @@ -367,6 +371,7 @@ build_ubuntu_cpu_cmake_no_tvm_op() { -DUSE_OPENCV=ON \ -DUSE_SIGNAL_HANDLER=ON \ -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja \ /work/mxnet @@ -519,6 +524,7 @@ build_ubuntu_cpu_mkldnn() { -DUSE_MKLDNN=ON \ -DUSE_CUDA=OFF \ -DUSE_CPP_PACKAGE=ON \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -534,6 +540,7 @@ build_ubuntu_cpu_mkldnn_mkl() { -DUSE_TVM_OP=ON \ -DUSE_MKL_IF_AVAILABLE=ON \ -DUSE_BLAS=MKL \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -GNinja /work/mxnet ninja } @@ -605,6 +612,7 @@ build_ubuntu_gpu_mkldnn() { -DUSE_CUDA=ON \ -DMXNET_CUDA_ARCH="$CI_CMAKE_CUDA_ARCH" \ -DUSE_CPP_PACKAGE=ON \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -619,6 +627,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { -DMXNET_CUDA_ARCH="$CI_CMAKE_CUDA_ARCH" \ -DUSE_CUDNN=OFF \ -DUSE_CPP_PACKAGE=ON \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } @@ -636,6 +645,7 @@ build_ubuntu_gpu_cuda101_cudnn7() { -DUSE_CPP_PACKAGE=ON \ -DUSE_DIST_KVSTORE=ON \ -DBUILD_CYTHON_MODULES=ON \ + -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \ -G Ninja /work/mxnet ninja } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 20b03e9f6b4a..3b8d2ca600e2 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -23,18 +23,18 @@ utils = load('ci/Jenkinsfile_utils.groovy') // mxnet libraries -mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' -mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so' +mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' +mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' mx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests' -mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so' +mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so' mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' +mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so' mx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.*, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h' diff --git a/tests/python/gpu/test_extensions_gpu.py b/tests/python/gpu/test_extensions_gpu.py index 1cc06cd8b2c5..9d3683166053 100644 --- a/tests/python/gpu/test_extensions_gpu.py +++ b/tests/python/gpu/test_extensions_gpu.py @@ -28,8 +28,8 @@ import pytest base_path = os.path.join(os.path.dirname(__file__), "../../..") -def check_platform(): - return platform.machine() not in ['x86_64', 'AMD64'] +def check_platform(supported_platforms=['x86_64', 'AMD64']): + return platform.machine() not in supported_platforms @pytest.mark.skipif(check_platform(), reason="not all machine types supported") @pytest.mark.skipif(is_cd_run(), reason="continuous delivery run - ignoring test") @@ -89,3 +89,24 @@ def test_custom_op_gpu(): mx.random.seed(128, ctx=mx.gpu()) r4 = mx.nd.my_noisy_relu(d2) assert_almost_equal(r3.asnumpy(), r4.asnumpy(), rtol=1e-3, atol=1e-3) + +@pytest.mark.skipif(check_platform(['x86_64']), reason="not all machine types supported") +@pytest.mark.skipif(is_cd_run(), reason="continuous delivery run - ignoring test") +def test_external_op(): + # check if operator already exists + if hasattr(mx.nd, 'min_ex'): + raise MXNetError('Operator already loaded') + + lib = 'libexternal_lib.so' + fname = os.path.join(base_path,'example/extensions/lib_external_ops/build/'+lib) + if not os.path.exists(fname): + raise MXNetError("library %s not found " % lib) + + fname = os.path.abspath(fname) + mx.library.load(fname, False) + + # execute operator + try: + mx.nd.min_ex() + except: + raise MXNetError('Operator not loaded successfully') diff --git a/tests/python/unittest/test_extensions.py b/tests/python/unittest/test_extensions.py index 52f999571e13..1c6763f5c515 100644 --- a/tests/python/unittest/test_extensions.py +++ b/tests/python/unittest/test_extensions.py @@ -28,8 +28,8 @@ import pytest base_path = os.path.join(os.path.dirname(__file__), "../../..") -def check_platform(): - return platform.machine() not in ['x86_64', 'AMD64'] +def check_platform(supported_platforms=['x86_64', 'AMD64']): + return platform.machine() not in supported_platforms @pytest.mark.skipif(check_platform(), reason="not all machine types supported") @pytest.mark.skipif(is_cd_run(), reason="continuous delivery run - ignoring test") @@ -179,3 +179,24 @@ def test_subgraph(): out5 = sym_block3(a_data, b_data) # check that result matches one executed by MXNet assert_almost_equal(out[0].asnumpy(), out5[0].asnumpy(), rtol=1e-3, atol=1e-3) + +@pytest.mark.skipif(check_platform(['x86_64']), reason="not all machine types supported") +@pytest.mark.skipif(is_cd_run(), reason="continuous delivery run - ignoring test") +def test_external_op(): + # check if operator already exists + if hasattr(mx.nd, 'min_ex'): + raise MXNetError('Operator already loaded') + + lib = 'libexternal_lib.so' + fname = os.path.join(base_path,'example/extensions/lib_external_ops/build/'+lib) + if not os.path.exists(fname): + raise MXNetError("library %s not found " % lib) + + fname = os.path.abspath(fname) + mx.library.load(fname, False) + + # execute operator + try: + mx.nd.min_ex() + except: + raise MXNetError('Operator not loaded successfully') From 0c9b4d83e567f7cdf7827d1598cb2293742c2aa4 Mon Sep 17 00:00:00 2001 From: Manu Seth Date: Fri, 6 Nov 2020 20:46:53 +0000 Subject: [PATCH 5/7] disable test on osx stage in ci --- .github/workflows/os_x_staticbuild.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/os_x_staticbuild.yml b/.github/workflows/os_x_staticbuild.yml index a93e580a8548..780e255bf083 100644 --- a/.github/workflows/os_x_staticbuild.yml +++ b/.github/workflows/os_x_staticbuild.yml @@ -44,6 +44,6 @@ jobs: - name: Test project run: | - python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial' - MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial' - python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial' + python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial' + MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial' + python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial' From 5bf96a5204271dd1e9da1ce8bed9f30e0b191442 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 Nov 2020 22:29:23 +0000 Subject: [PATCH 6/7] cleaned up example CMakeLists.txt removed -shared from GPU --- example/extensions/lib_external_ops/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/example/extensions/lib_external_ops/CMakeLists.txt b/example/extensions/lib_external_ops/CMakeLists.txt index cebf8b79c4b8..94552e55e8d6 100644 --- a/example/extensions/lib_external_ops/CMakeLists.txt +++ b/example/extensions/lib_external_ops/CMakeLists.txt @@ -11,7 +11,6 @@ FILE(GLOB CXX_SRCS add_library(external_lib SHARED ${CXX_SRCS}) target_link_libraries(external_lib PUBLIC mxnet) -# generic GPU stuff if(USE_CUDA) # specify GPU sources (optional) FILE(GLOB CU_SRCS "*.cu") @@ -19,9 +18,12 @@ if(USE_CUDA) endif(USE_CUDA) if(UNIX) - # unix-specific stuff + # unix-specific build configuration if(USE_CUDA) - # unix+GPU-specific stuff - target_compile_options(external_lib PUBLIC -shared) + # unix+GPU-specific build configuration endif(USE_CUDA) endif() + +if(MSVC) + message(FATAL_ERROR "Windows builds are not support for external ops") +endif() From 76abb738579b45f975a1fcfab6ce3b4f5d290d82 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 13 Nov 2020 01:06:31 +0000 Subject: [PATCH 7/7] moved windows check --- CMakeLists.txt | 6 +++++- example/extensions/lib_external_ops/CMakeLists.txt | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ffda36f3a4d..30839b45c339 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -789,7 +789,11 @@ add_library(subgraph_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/l add_library(pass_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_pass/pass_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc) if(IS_DIRECTORY ${BUILD_EXTENSION_PATH}) - add_subdirectory(${BUILD_EXTENSION_PATH} ${BUILD_EXTENSION_PATH}/build) + if(MSVC) + message(FATAL_ERROR "Windows builds are not support for external ops") + else() + add_subdirectory(${BUILD_EXTENSION_PATH} ${BUILD_EXTENSION_PATH}/build) + endif() endif() target_include_directories(customop_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet) diff --git a/example/extensions/lib_external_ops/CMakeLists.txt b/example/extensions/lib_external_ops/CMakeLists.txt index 94552e55e8d6..383e5298f074 100644 --- a/example/extensions/lib_external_ops/CMakeLists.txt +++ b/example/extensions/lib_external_ops/CMakeLists.txt @@ -16,14 +16,3 @@ if(USE_CUDA) FILE(GLOB CU_SRCS "*.cu") target_sources(external_lib PUBLIC ${CU_SRCS}) endif(USE_CUDA) - -if(UNIX) - # unix-specific build configuration - if(USE_CUDA) - # unix+GPU-specific build configuration - endif(USE_CUDA) -endif() - -if(MSVC) - message(FATAL_ERROR "Windows builds are not support for external ops") -endif()