Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
import Julia binding
Browse files Browse the repository at this point in the history
- enable Jenkins CI build for Julia
- add license headers to Julia source code
- update links for Julia README
  • Loading branch information
iblislin committed Sep 28, 2018
1 parent 0a04594 commit e190c9a
Show file tree
Hide file tree
Showing 79 changed files with 1,371 additions and 29 deletions.
44 changes: 35 additions & 9 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,30 @@ def python3_gpu_ut_nocudnn(docker_container_name) {
}
}

def deploy_docs() {
parallel 'Docs': {
node(NODE_LINUX_CPU) {
ws('workspace/docs') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
}
}
}
},
'Julia docs': {
node(NODE_LINUX_CPU) {
ws('workspace/julia-docs') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_lib)
utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false)
}
}
}
}
}

node('mxnetlinux-cpu') {
// Loading the utilities requires a node context unfortunately
checkout scm
Expand Down Expand Up @@ -746,6 +770,16 @@ core_logic: {
}
}
},
'Julia 0.6: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-julia06-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_lib)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06', false)
}
}
}
},

'Python 2: CPU Win':{
node(NODE_WINDOWS_CPU) {
Expand Down Expand Up @@ -911,15 +945,7 @@ core_logic: {
}

stage('Deploy') {
node(NODE_LINUX_CPU) {
ws('workspace/docs') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'deploy_docs', false)
sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
}
}
}
deploy_docs()
}
}
,
Expand Down
3 changes: 3 additions & 0 deletions ci/docker/Dockerfile.build.ubuntu_cpu
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ RUN /work/ubuntu_r.sh
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh

COPY install/ubuntu_julia.sh /work/
RUN /work/ubuntu_julia.sh

COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh

Expand Down
37 changes: 37 additions & 0 deletions ci/docker/install/ubuntu_julia.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# build and install are separated so changes to build don't invalidate
# the whole docker cache for the image

set -ex

export JLBINARY='julia.tar.gz'
export JULIADIR='/work/julia'
export JULIA="${JULIADIR}/bin/julia"

mkdir -p $JULIADIR
# The julia version in Ubuntu repo is too old
# We download the tarball from the official link:
# https://julialang.org/downloads/
wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.2-linux-x86_64.tar.gz
tar xzvf $JLBINARY -C $JULIADIR --strip 1
rm $JLBINARY

$JULIA -e 'versioninfo()'
54 changes: 54 additions & 0 deletions ci/docker/runtime_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,35 @@ unittest_ubuntu_gpu_R() {
make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
}

unittest_ubuntu_cpu_julia06() {
set -ex
export PATH="/work/julia/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export JULIA_PKGDIR='/work/julia-pkg'
export DEPDIR=`julia -e 'print(Pkg.dir())'`

julia -e 'versioninfo()'
julia -e 'Pkg.init()'

# install package
ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet

# install dependencies
julia -e 'Pkg.resolve()'

# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'

# use the prebuilt binary from $MXNET_HOME/lib
julia -e 'Pkg.build("MXNet")'

# run the script `julia/test/runtests.jl`
julia -e 'Pkg.test("MXNet")'

# See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774
julia -e 'using MXNet; mx._sig_checker()'
}

unittest_centos7_cpu() {
set -ex
cd /work/mxnet
Expand Down Expand Up @@ -1090,6 +1119,31 @@ deploy_docs() {
popd
}

deploy_jl_docs() {
set -ex
export PATH="/work/julia/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export JULIA_PKGDIR='/work/julia-pkg'
export DEPDIR=`julia -e 'print(Pkg.dir())'`

julia -e 'versioninfo()'
julia -e 'Pkg.init()'
ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet
julia -e 'Pkg.resolve()'

# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'

# use the prebuilt binary from $MXNET_HOME/lib
julia -e 'Pkg.build("MXNet")'
# build docs
julia -e 'Pkg.add("Documenter")'
julia -e 'cd(Pkg.dir("MXNet")); include(joinpath("docs", "make.jl"))'

# TODO: make Jenkins worker push to MXNet.jl ph-pages branch if master build
# ...
}

# broken_link_checker

broken_link_checker() {
Expand Down
15 changes: 3 additions & 12 deletions julia/README.md
Original file line number Diff line number Diff line change
@@ -1,23 +1,14 @@
# MXNet

[![Build Status](https://travis-ci.org/dmlc/MXNet.jl.svg?branch=master)](https://travis-ci.org/dmlc/MXNet.jl)
[![Windows Build](https://ci.appveyor.com/api/projects/status/re90njols2th2ide?svg=true)](https://ci.appveyor.com/project/pluskid/mxnet-jl)
[![codecov.io](https://codecov.io/github/dmlc/MXNet.jl/coverage.svg?branch=master)](https://codecov.io/github/dmlc/MXNet.jl?branch=master)
[![](https://img.shields.io/badge/docs-latest-blue.svg)](https://dmlc.github.io/MXNet.jl/latest)
[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://dmlc.github.io/MXNet.jl/stable)
[![MXNet](http://pkg.julialang.org/badges/MXNet_0.6.svg)](http://pkg.julialang.org/?pkg=MXNet)
[![License](http://dmlc.github.io/img/apache2.svg)](LICENSE.md)
[![Join the chat at https://gitter.im/dmlc/mxnet](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dmlc/mxnet)


MXNet.jl is the [dmlc/mxnet](https://github.com/dmlc/mxnet) [Julia](http://julialang.org/) package. MXNet.jl brings flexible and efficient GPU computing and state-of-art deep learning to Julia. Some highlight of its features include:
MXNet.jl is the [dmlc/mxnet](https://github.com/apache/incubator-mxnet) [Julia](http://julialang.org/) package. MXNet.jl brings flexible and efficient GPU computing and state-of-art deep learning to Julia. Some highlight of its features include:

* Efficient tensor/matrix computation across multiple devices, including multiple CPUs, GPUs and distributed server nodes.
* Flexible symbolic manipulation to composite and construct state-of-the-art deep learning models.
* Flexible symbolic manipulation to composite and construction of state-of-the-art deep learning models.

The current support julia version is 0.6. Julia 0.7/1.0 is not supported yet.

Here is an example of how training a simple 3-layer MLP on MNIST looks like:
Here is an example of how training a simple 3-layer MLP on MNIST:

```julia
using MXNet
Expand Down
17 changes: 17 additions & 0 deletions julia/appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

environment:
matrix:
- JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
Expand Down
21 changes: 19 additions & 2 deletions julia/deps/build.jl
Original file line number Diff line number Diff line change
@@ -1,11 +1,28 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import JSON

################################################################################
# First try to detect and load existing libmxnet
################################################################################
libmxnet_detected = false
libmxnet_curr_ver = get(ENV, "MXNET_COMMIT", "master")
curr_win = "20171019" # v0.12.0
curr_win = "20180211" # v1.1.0

if haskey(ENV, "MXNET_HOME")
MXNET_HOME = ENV["MXNET_HOME"]
Expand Down Expand Up @@ -171,7 +188,7 @@ if !libmxnet_detected
@build_steps begin
BinDeps.DirectoryRule(_mxdir, @build_steps begin
ChangeDirectory(_srcdir)
`git clone https://github.com/dmlc/mxnet`
`git clone https://github.com/apache/incubator-mxnet`
end)
@build_steps begin
ChangeDirectory(_mxdir)
Expand Down
19 changes: 19 additions & 0 deletions julia/deps/cblas.h
Original file line number Diff line number Diff line change
@@ -1,3 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#ifndef CBLAS_H
#define CBLAS_H

Expand Down
18 changes: 18 additions & 0 deletions julia/deps/cpcblas.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,23 @@
#!/bin/sh

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.


# be invoked from build.jl

set -e
Expand Down
17 changes: 17 additions & 0 deletions julia/docs/Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

all:
julia --color=yes ./make.jl
mkdocs build
17 changes: 17 additions & 0 deletions julia/docs/make.jl
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

using Documenter, MXNet

makedocs(
Expand Down
17 changes: 17 additions & 0 deletions julia/docs/mkdocs.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

site_name: MXNet.jl
repo_url: https://github.com/dmlc/MXNet.jl

Expand Down
17 changes: 17 additions & 0 deletions julia/examples/char-lstm/config.jl
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

const DROPOUT = 0
const BATCH_SIZE = 32
const SEQ_LENGTH = 32
Expand Down
Loading

0 comments on commit e190c9a

Please sign in to comment.