Skip to content

Commit 6ea2152

Browse files
author
Ubuntu
committed
[New] branch from marsan/docker_mldm
0 parents  commit 6ea2152

12 files changed

+1036
-0
lines changed

Diff for: ‎0-prepare-host.sh

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/bash
2+
3+
#--------------------------------
4+
# Install docker basic version
5+
#--------------------------------
6+
# configure prerequisites
7+
sudo apt-get update
8+
sudo apt-get install --assume-yes wget
9+
10+
# install docker
11+
wget -qO- https://get.docker.com/ | sh
12+
13+
# configure docker (i.e. force devicemapper and bind to localhost)
14+
sudo ln -sf /usr/bin/docker /usr/local/bin/docker
15+
sudo sed -i '$acomplete -F _docker docker' /etc/bash_completion.d/docker
16+
#sudo sed -i 's|^#*DOCKER_OPTS.*$|DOCKER_OPTS="-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock"|g' /etc/default/docker
17+
sudo sed -i 's|^#*DOCKER_OPTS.*$|DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4 -g /home/$(whoami)/workspace/dockers/images"|g' /etc/default/docker
18+
19+
# create docker group and add user
20+
sudo groupadd docker
21+
sudo usermod -aG docker $(whoami)
22+
23+
# install docker-compose (https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-14-04)
24+
sudo apt-get -y install python-pip
25+
sudo pip install docker-compose
26+
27+
#--------------------------------
28+
# GPU for deep learning
29+
# (nvidia driver, cuda, nvidia-docker)
30+
#--------------------------------
31+
# nvidia driver dependency
32+
sudo apt-get update
33+
sudo apt-get install -y gcc make
34+
35+
# nvidia driver drm workaround (https://github.com/BVLC/caffe/wiki/Caffe-on-EC2-Ubuntu-14.04-Cuda-7)
36+
sudo apt-get install -y linux-image-extra-`uname -r` linux-headers-`uname -r` linux-image-`uname -r`
37+
38+
# install nvidia driver
39+
wget http://us.download.nvidia.com/XFree86/Linux-x86_64/367.27/NVIDIA-Linux-x86_64-367.27.run
40+
chmod +x NVIDIA-Linux-x86_64-367.27.run
41+
sudo ./NVIDIA-Linux-x86_64-367.27.run -a -x --ui=none
42+
43+
# install cuda repository
44+
wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/cuda-repo-ubuntu1404_7.5-18_amd64.deb
45+
sudo dpkg -i cuda-repo-ubuntu1404_7.5-18_amd64.deb
46+
47+
#update apt
48+
sudo apt-get update
49+
sudo apt-get upgrade -y
50+
sudo apt-get install -y opencl-headers build-essential protobuf-compiler \
51+
libprotoc-dev libboost-all-dev libleveldb-dev hdf5-tools libhdf5-serial-dev \
52+
libopencv-core-dev libopencv-highgui-dev libsnappy-dev libsnappy1 \
53+
libatlas-base-dev cmake libstdc++6-4.8-dbg libgoogle-glog0 libgoogle-glog-dev \
54+
libgflags-dev liblmdb-dev git python-pip gfortran
55+
sudo apt-get clean
56+
57+
# install cuda
58+
sudo apt-get install -y cuda
59+
sudo apt-get clean
60+
61+
# verify cuda
62+
nvidia-smi
63+
lsmod | grep -i nvidia
64+
ls -alh /dev | grep -i nvidia
65+
66+
# Install nvidia-docker and nvidia-docker-plugin
67+
wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.0-rc.3/nvidia-docker_1.0.0.rc.3-1_amd64.deb
68+
sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb
69+
70+
# Test nvidia-smi
71+
nvidia-docker run --rm nvidia/cuda nvidia-smi
72+
73+

Diff for: ‎1-build-mldm.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
docker build -t marsan/mldm_gpu .
4+
5+
#docker-compose build

Diff for: ‎2-start-mldm.sh

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
WORK=$HOME/workspace
4+
5+
nvidia-docker run -d \
6+
--name=mldm_gpu \
7+
-v $WORK:/home/$(whoami)/workspace \
8+
-p 8880-8899:8880-8899 \
9+
-it marsan/mldm_gpu
10+
11+
# --device /dev/nvidia0:/dev/nvidia0 \
12+
# --device /dev/nvidia1:/dev/nvidia1 \
13+
# --device /dev/nvidia2:/dev/nvidia2 \
14+
# --device /dev/nvidia3:/dev/nvidia3 \
15+
# --device /dev/nvidiactl:/dev/nvidiactl \
16+
# --device /dev/nvidia-uvm:/dev/nvidia-uvm \
17+
18+
19+
## -h mldm_env \
20+
## -p 6800-6810:6800-6810 \
21+
##docker run -it kaggle/python bash
22+
23+
#docker-compose up -d
24+
#docker-compose restart
25+

Diff for: ‎Dockerfile

+162
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# machine learning gears
2+
FROM gcr.io/tensorflow/tensorflow:latest-devel-gpu
3+
MAINTAINER Marsan Ma <[email protected]>
4+
5+
#---------------------------------
6+
# basic tools
7+
#---------------------------------
8+
RUN apt-get update
9+
RUN apt-get install -y wget htop vim unzip procps
10+
11+
#---------------------------------
12+
# kaggle/python layers
13+
#---------------------------------
14+
# Anaconda3 (https://hub.docker.com/r/continuumio/anaconda3/~/dockerfile/)
15+
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
16+
17+
RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \
18+
libglib2.0-0 libxext6 libsm6 libxrender1 \
19+
git mercurial subversion
20+
21+
RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
22+
wget --quiet https://repo.continuum.io/archive/Anaconda3-4.1.0-Linux-x86_64.sh -O ~/anaconda.sh && \
23+
/bin/bash ~/anaconda.sh -b -p /opt/conda && \
24+
rm ~/anaconda.sh
25+
26+
RUN apt-get install -y curl grep sed dpkg && \
27+
TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \
28+
curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
29+
dpkg -i tini.deb && \
30+
rm tini.deb && \
31+
apt-get clean
32+
33+
ENV PATH /opt/conda/bin:$PATH
34+
35+
# kaggle/python0 (https://hub.docker.com/r/kaggle/python0/~/dockerfile/)
36+
RUN apt-get install -y build-essential python-software-properties
37+
38+
# kaggle/python1 (https://hub.docker.com/r/kaggle/python1/~/dockerfile/)
39+
RUN pip install seaborn python-dateutil spacy dask pytagcloud pyyaml ggplot joblib \
40+
husl geopy ml_metrics mne pyshp gensim && \
41+
apt-get install -y libfreetype6-dev && \
42+
apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \
43+
# textblob
44+
pip install textblob && \
45+
#word cloud
46+
pip install git+git://github.com/amueller/word_cloud.git
47+
48+
# kaggle/python2 (https://hub.docker.com/r/kaggle/python2/~/dockerfile/)
49+
RUN apt-get install -y libfreetype6-dev && \
50+
apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \
51+
# textblob
52+
# pip install python-igraph && \
53+
#xgboost
54+
cd /usr/local/src && mkdir xgboost && cd xgboost && \
55+
git clone --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \
56+
make && cd python-package && python setup.py install
57+
58+
# kaggle/python3 (https://hub.docker.com/r/kaggle/python3/~/dockerfile/)
59+
RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \
60+
pip install Pillow
61+
62+
# kaggle/python (https://hub.docker.com/r/kaggle/python/~/dockerfile/)
63+
#RUN cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn.git && \
64+
# cd scikit-learn && python setup.py build && python setup.py install && \
65+
RUN conda install scikit-learn
66+
# HDF5 support
67+
RUN conda install h5py
68+
69+
RUN pip install --upgrade mpld3 && \
70+
pip install mplleaflet && \
71+
pip install gpxpy && \
72+
pip install arrow && \
73+
pip install sexmachine && \
74+
pip install Geohash && \
75+
pip install deap && \
76+
pip install tpot && \
77+
pip install haversine && \
78+
pip install toolz cytoolz && \
79+
pip install sacred && \
80+
pip install plotly && \
81+
pip install git+https://github.com/nicta/dora.git && \
82+
pip install git+https://github.com/hyperopt/hyperopt.git && \
83+
# tflean. Deep learning library featuring a higher-level API for TensorFlow. http://tflearn.org
84+
#pip install git+https://github.com/tflearn/tflearn.git && \
85+
pip install fitter && \
86+
pip install langid && \
87+
# Delorean. Useful for dealing with datetime
88+
pip install delorean
89+
90+
#---------------------------------
91+
# Marsan's toolbelt
92+
#---------------------------------
93+
RUN conda install mkl
94+
RUN conda install libgfortran
95+
96+
RUN pip install -U \
97+
pandas \
98+
mongoengine \
99+
bottle \
100+
cherrypy \
101+
jieba3k \
102+
yolk3k \
103+
azure \
104+
cython \
105+
html5lib \
106+
pyyaml \
107+
demjson \
108+
hanziconv \
109+
ftfy \
110+
hiredis \
111+
google-api-python-client
112+
113+
RUN pip install -U \
114+
Django \
115+
django-pipeline \
116+
django-bootstrap3 \
117+
django_compressor \
118+
rest-pandas \
119+
gunicorn \
120+
boto3 \
121+
PyMySQL
122+
123+
RUN pip install \
124+
djangoajax \
125+
django-dashing
126+
127+
128+
# MySQL
129+
RUN apt-get install -y python3-dev libmysqlclient-dev
130+
RUN pip install mysqlclient
131+
132+
133+
# pathos (python parallel process)
134+
#RUN pip install -U git+https://github.com/uqfoundation/pathos.git@master
135+
136+
#RUN pip install -U \
137+
# newspaper3k
138+
139+
# Tensorflow GPU supported version
140+
RUN pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl
141+
142+
#---------------------------------
143+
# Enviroment
144+
#---------------------------------
145+
# Timezone
146+
RUN echo "Asia/Taipei" > /etc/timezone
147+
RUN dpkg-reconfigure -f noninteractive tzdata
148+
149+
# Add runner script
150+
COPY files/runner.sh /runner.sh
151+
RUN chmod +x /runner.sh
152+
COPY files/bashrc /.bashrc
153+
COPY files/vimrc /.vimrc
154+
155+
## Set the working directory
156+
WORKDIR /home/workspace
157+
RUN mkdir /home/workspace/notebooks
158+
#VOLUME /Users/marsan/wordspace
159+
160+
EXPOSE 8880:8900
161+
162+
ENTRYPOINT ["/runner.sh"]

0 commit comments

Comments
 (0)