Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

julia: rename build env var MXNET_HOME to MXNET_ROOT #15568

Merged
merged 11 commits into from
Sep 8, 2019
8 changes: 4 additions & 4 deletions ci/docker/runtime_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1088,7 +1088,7 @@ unittest_ubuntu_gpu_R() {
unittest_ubuntu_cpu_julia() {
set -ex
export PATH="$1/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export MXNET_ROOT='/work/mxnet'
export JULIA_DEPOT_PATH='/work/julia-depot'
export INTEGRATION_TEST=1

Expand All @@ -1098,7 +1098,7 @@ unittest_ubuntu_cpu_julia() {
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH

# use the prebuilt binary from $MXNET_HOME/lib
# use the prebuilt binary from $MXNET_ROOT/lib
julia --project=./julia -e 'using Pkg; Pkg.build("MXNet")'

# run the script `julia/test/runtests.jl`
Expand Down Expand Up @@ -1253,7 +1253,7 @@ build_docs() {

# Setup environment for Julia docs
export PATH="/work/julia10/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export MXNET_ROOT='/work/mxnet'
export JULIA_DEPOT_PATH='/work/julia-depot'

julia -e 'using InteractiveUtils; versioninfo()'
Expand Down Expand Up @@ -1465,7 +1465,7 @@ deploy_docs() {

# Setup for Julia docs
export PATH="/work/julia10/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export MXNET_ROOT='/work/mxnet'
export JULIA_DEPOT_PATH='/work/julia-depot'

julia -e 'using InteractiveUtils; versioninfo()'
Expand Down
2 changes: 1 addition & 1 deletion ci/windows/test_jl07_cpu.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# set default output encoding to utf8
$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'

$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
$env:MXNET_ROOT = [System.IO.Path]::GetFullPath('.\windows_package')
$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7.0-win64.exe"
$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')

Expand Down
2 changes: 1 addition & 1 deletion ci/windows/test_jl10_cpu.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# set default output encoding to utf8
$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'

$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
$env:MXNET_ROOT = [System.IO.Path]::GetFullPath('.\windows_package')
$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0.3-win64.exe"
$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')

Expand Down
7 changes: 7 additions & 0 deletions julia/NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@

# v1.6.0

* Rename environment variable `MXNET_HOME` to `MXNET_ROOT` (#15568).

* Environment variable `MXNET_LIBRARY_PATH` supports (#15568).

```shell
$ MXNET_LIBRARY_PATH=/path/to/libmxnet.so julia
```

# v1.5.0

Expand Down
33 changes: 27 additions & 6 deletions julia/deps/build.jl
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,35 @@ libmxnet_detected = false
libmxnet_curr_ver = get(ENV, "MXNET_COMMIT", "master")
curr_win = "20190608" # v1.5.0

# TODO: remove MXNET_HOME backward compatibility in v1.7
if haskey(ENV, "MXNET_HOME")
MXNET_HOME = ENV["MXNET_HOME"]
@info("MXNET_HOME environment detected: $MXNET_HOME")
@warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead."
end

# TODO: remove MXNET_HOME backward compatibility in v1.7
iblislin marked this conversation as resolved.
Show resolved Hide resolved
MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", ""))
search_locations = if !isempty(MXNET_ROOT)
!isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path")
@info "env var: MXNET_ROOT -> $MXNET_ROOT"
[joinpath(MXNET_ROOT, "lib"), MXNET_ROOT]
else
[]
end

MXNET_LIBRARY_PATH = get(ENV, "MXNET_LIBRARY_PATH", "")
println(typeof(MXNET_LIBRARY_PATH))
# In case of macOS, if user build libmxnet from source and set the MXNET_ROOT,
# the output is still named as `libmxnet.so`.
search_names = ["libmxnet.$(Libdl.dlext)", "libmxnet.so"]
if !isempty(MXNET_LIBRARY_PATH)
!isabspath(MXNET_LIBRARY_PATH) && error("MXNET_LIBRARY_PATH should be a absolute path")
@info "env var: MXNET_LIBRARY_PATH -> $MXNET_LIBRARY_PATH"
pushfirst!(search_names, MXNET_LIBRARY_PATH)
end

if (!isempty(MXNET_ROOT)) || (!isempty(MXNET_LIBRARY_PATH))
@info("Trying to load existing libmxnet...")
# In case of macOS, if user build libmxnet from source and set the MXNET_HOME,
# the output is still named as `libmxnet.so`.
lib = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"],
[joinpath(MXNET_HOME, "lib"), MXNET_HOME])
lib = Libdl.find_library(search_names, search_locations)
if !isempty(lib)
@info("Existing libmxnet detected at $lib, skip building...")
libmxnet_detected = true
Expand Down
20 changes: 12 additions & 8 deletions julia/docs/src/user-guide/install.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,13 @@ libmxnet.

There are several environment variables that change this behaviour.

- `MXNET_HOME`: If you already have a pre-installed version of mxnet
you can use `MXNET_HOME` to point the build-process in the right direction.
- `CUDA_HOME`: If the automatic cuda detection fails you can also set `CUDA_HOME`
- `MXNET_ROOT`: If you already have a pre-installed version of mxnet
you can use `MXNET_ROOT` to point the build-process in the right direction.
Note that you should set this variable as a absolute path.
- `MXNET_HOME`: This variable is replaced by `MXNET_ROOT`. It has been renamed
as of v1.6.
- `MXNET_LIBRARY_PATH`: The absolute path of the libmxnet share object.
- `CUDA_HOME`: If the automatic CUDA detection fails you can also set `CUDA_HOME`
to override the process.
- `MXNET_COMMIT`: To control which version of libmxnet will be compiled,
you can use the`MXNET_COMMIT` variable to point to either a version tag
Expand Down Expand Up @@ -80,26 +84,26 @@ to work with a separate, maybe customized libmxnet.

To build libmxnet, please refer to [the installation guide of
libmxnet](https://mxnet.incubator.apache.org/install/index.html). After
successfully installing libmxnet, set the `MXNET_HOME` *environment
successfully installing libmxnet, set the `MXNET_ROOT` *environment
variable* to the location of libmxnet. In other words, the compiled
`libmxnet.so` should be found in `$MXNET_HOME/lib`.
`libmxnet.so` should be found in `$MXNET_ROOT/lib`.

> **note**
>
> The constant `MXNET_HOME` is pre-compiled in MXNet.jl package cache.
> The constant `MXNET_ROOT` is pre-compiled in MXNet.jl package cache.
> If you updated the environment variable after installing MXNet.jl,
> make sure to update the pre-compilation cache by
> `Base.compilecache("MXNet")`.

When the `MXNET_HOME` environment variable is detected and the
When the `MXNET_ROOT` environment variable is detected and the
corresponding `libmxnet.so` could be loaded successfully, MXNet.jl will
skip automatic building during installation and use the specified
libmxnet instead.

Basically, MXNet.jl will search `libmxnet.so` or `libmxnet.dll` in the
following paths (and in that order):

- `$MXNET_HOME/lib`: customized libmxnet builds
- `$MXNET_ROOT/lib`: customized libmxnet builds
- `Pkg.dir("MXNet", "deps", "usr", "lib")`: automatic builds
- Any system wide library search path

Expand Down
30 changes: 25 additions & 5 deletions julia/src/base.jl
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,31 @@ const grad_req_map = Dict{Symbol,GRAD_REQ}(
################################################################################
# Initialization and library API entrance
################################################################################
const MXNET_LIB = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"], # see build.jl
[joinpath(get(ENV, "MXNET_HOME", ""), "lib"),
get(ENV, "MXNET_HOME", ""),
joinpath(@__DIR__, "..",
"deps", "usr", "lib")])
function _get_search_names()
MXNET_LIBRARY_PATH = get(ENV, "MXNET_LIBRARY_PATH", "")
A = ["libmxnet.$(Libdl.dlext)", "libmxnet.so"] # see build.jl
if !isempty(MXNET_LIBRARY_PATH)
!isabspath(MXNET_LIBRARY_PATH) && error("MXNET_LIBRARY_PATH should be a absolute path")
pushfirst!(A, MXNET_LIBRARY_PATH)
end
A
end

function _get_search_dirs()
# TODO: remove MXNET_HOME backward compatibility in v1.7
if haskey(ENV, "MXNET_HOME")
@warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead."
end
A = [joinpath(@__DIR__, "..", "deps", "usr", "lib")]
MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", ""))
if !isempty(MXNET_ROOT)
!isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path")
prepend!(A, [joinpath(MXNET_ROOT, "lib"), MXNET_ROOT])
end
A
end

const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs())
const LIB_VERSION = Ref{Cint}(0)

if isempty(MXNET_LIB)
Expand Down