Skip to content

Commit 560309e

Browse files
acezenacezen
authored
add scripts to install dependencies and build GraphScope (#222)
1 parent 1c8739e commit 560309e

File tree

3 files changed

+281
-3
lines changed

3 files changed

+281
-3
lines changed

‎docs/deployment.rst

+23-3
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ Install Chart
5353
Check Service Availability
5454

5555
.. code:: bash
56-
57-
# Helm 3 or 2
56+
57+
# Helm 3 or 2
5858
$ helm test [RELEASE_NAME]
5959
6060
Find `more details <https://github.com/alibaba/GraphScope/blob/main/charts/graphscope/README.md>`_ on how to connect a pre-launched service in python client.
@@ -76,4 +76,24 @@ You can use the script as follows or use `./script/launch_cluster.py --help` to
7676
.. code:: shell
7777
7878
pip3 install click PyYAML alibabacloud_cs20151215 alibabacloud_ecs20140526 alibabacloud_vpc20160428
79-
./scripts/launch_cluster.py --type aliyun --id your_access_key_id --secret your_access_key_secret --region your_region_id --output kube_config_path
79+
./scripts/launch_cluster.py --type aliyun --id your_access_key_id --secret your_access_key_secret --region your_region_id --output kube_config_path
80+
81+
82+
Deployment on local
83+
----------------------
84+
we provide scripts to install dependencies and build GraphScope on local.
85+
86+
* install independencies
87+
.. code:: shell
88+
89+
./script/install_denpendencies.sh
90+
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
91+
export PATH=${JAVA_HOME}/bin:${PATH}:/usr/local/go/bin:/usr/local/zookeeper/bin:/usr/share/maven/bin
92+
export GRAPHSCOPE_PREFIX=/tmp/graphscope_prefix
93+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
94+
95+
* build GraphScope on local
96+
.. code:: shell
97+
98+
./script/build.sh
99+

‎scripts/build.sh

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/usr/bin/env bash
2+
#
3+
# A script to build GraphScope standalone.
4+
5+
set -e
6+
set -x
7+
set -o pipefail
8+
9+
graphscope_src="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )"
10+
11+
function install_libgrape-lite() {
12+
echo "$(date '+%Y-%m-%d %H:%M:%S') build and install libgrape-lite"
13+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
14+
git clone -b master --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
15+
pushd /tmp/libgrape-lite
16+
mkdir build && cd build
17+
cmake ..
18+
make -j`nproc`
19+
sudo make install
20+
popd
21+
rm -fr /tmp/libgrape-lite
22+
}
23+
24+
function install_vineyard() {
25+
echo "$(date '+%Y-%m-%d %H:%M:%S') build and install vineyard"
26+
git clone -b v0.1.14 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git /tmp/libvineyard
27+
pushd /tmp/libvineyard
28+
git submodule update --init
29+
mkdir build && pushd build
30+
cmake .. -DBUILD_VINEYARD_PYPI_PACKAGES=ON -DBUILD_SHARED_LIBS=ON -DBUILD_VINEYARD_IO_OSS=ON
31+
make -j`nproc`
32+
make vineyard_client_python -j`nproc`
33+
sudo make install
34+
popd
35+
python3 setup.py bdist_wheel
36+
pip3 install -U ./dist/*.whl
37+
popd
38+
rm -fr /tmp/libvineyard
39+
}
40+
41+
function build_graphscope_gae() {
42+
echo "$(date '+%Y-%m-%d %H:%M:%S') build and install graphscope analytical engine"
43+
# build GraphScope GAE
44+
cd ${graphscope_src}
45+
mkdir analytical_engine/build && pushd analytical_engine/build
46+
cmake ..
47+
make -j`nproc`
48+
sudo make install
49+
popd
50+
}
51+
52+
function build_graphscope_gie() {
53+
echo "$(date '+%Y-%m-%d %H:%M:%S') build and install graphscope interactive engine"
54+
# build GraphScope GIE
55+
source ~/.cargo/env
56+
cd ${graphscope_src}
57+
# build frontend coordinator graph-manager
58+
pushd interactive_engine
59+
mvn clean package -DskipTests -Pjava-release --quiet
60+
popd
61+
# build executor
62+
pushd interactive_engine/src/executor
63+
cargo build --all --release
64+
popd
65+
# copy dependencies into GRAPHSCOPE_PREFIX
66+
mkdir -p ${GRAPHSCOPE_PREFIX}/pid ${GRAPHSCOPE_PREFIX}/logs
67+
# copy mvn package
68+
cp ./interactive_engine/src/instance-manager/target/0.0.1-SNAPSHOT.tar.gz ${GRAPHSCOPE_PREFIX}/0.0.1-instance-manager-SNAPSHOT.tar.gz
69+
cp ./interactive_engine/src/assembly/target/0.0.1-SNAPSHOT.tar.gz ${GRAPHSCOPE_PREFIX}/0.0.1-SNAPSHOT.tar.gz
70+
tar -xf ${GRAPHSCOPE_PREFIX}/0.0.1-SNAPSHOT.tar.gz -C ${GRAPHSCOPE_PREFIX}/
71+
tar -xf ${GRAPHSCOPE_PREFIX}/0.0.1-instance-manager-SNAPSHOT.tar.gz -C ${GRAPHSCOPE_PREFIX}/
72+
# coordinator
73+
mkdir -p ${GRAPHSCOPE_PREFIX}/coordinator
74+
cp -r ./interactive_engine/src/coordinator/target ${GRAPHSCOPE_PREFIX}/coordinator/
75+
# frontend
76+
mkdir -p ${GRAPHSCOPE_PREFIX}/frontend/frontendservice
77+
cp -r ./interactive_engine/src/frontend/frontendservice/target ${GRAPHSCOPE_PREFIX}/frontend/frontendservice/
78+
# executor
79+
mkdir -p ${GRAPHSCOPE_PREFIX}/conf
80+
cp ./interactive_engine/src/executor/target/release/executor ${GRAPHSCOPE_PREFIX}/bin/executor
81+
cp ./interactive_engine/src/executor/store/log4rs.yml ${GRAPHSCOPE_PREFIX}/conf/log4rs.yml
82+
}
83+
84+
function build_graphscope_gle() {
85+
echo "$(date '+%Y-%m-%d %H:%M:%S') build and install graphscope learning engine"
86+
cd ${graphscope_src}
87+
git submodule update --init
88+
pushd learning_engine/graph-learn
89+
git submodule update --init third_party/pybind11
90+
mkdir cmake-build && pushd cmake-build
91+
cmake .. -DWITH_VINEYARD=ON -DTESTING=OFF
92+
make -j`nproc`
93+
sudo make install
94+
popd
95+
}
96+
97+
function install_client_and_coordinator() {
98+
echo "$(date '+%Y-%m-%d %H:%M:%S') install graphscope coordinator and client"
99+
# install GraphScope client
100+
export WITH_LEARNING_ENGINE=ON
101+
cd ${graphscope_src}
102+
pushd python
103+
pip3 install -U setuptools
104+
pip3 install -r requirements.txt -r requirements-dev.txt
105+
python3 setup.py bdist_wheel
106+
pip3 install -U ./dist/*.whl
107+
popd
108+
109+
# install GraphScope coordinator
110+
pushd coordinator
111+
pip3 install -r requirements.txt -r requirements-dev.txt
112+
python3 setup.py bdist_wheel
113+
pip3 install -U ./dist/*.whl
114+
popd
115+
}
116+
117+
install_libgrape-lite
118+
119+
install_vineyard
120+
121+
build_graphscope_gae
122+
123+
build_graphscope_gie
124+
125+
build_graphscope_gle
126+
127+
install_client_and_coordinator
128+
129+
echo "The script has successfully builded GraphScope."
130+
echo "Now you are ready to have fun with GraphScope."
131+
132+
set +x
133+
set +e
134+
set +o pipefail

‎scripts/install_dependencies.sh

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/usr/bin/env bash
2+
#
3+
# A script to install dependencies of GraphScope.
4+
5+
set -e
6+
set -x
7+
set -o pipefail
8+
9+
is_in_wsl=false && [[ ! -z "${IS_WSL}" || ! -z "${WSL_DISTRO_NAME}" ]] && is_in_wsl=true
10+
11+
# https://unix.stackexchange.com/questions/6345/how-can-i-get-distribution-name-and-version-number-in-a-simple-shell-script
12+
function get_os_version() {
13+
if [ -f /etc/os-release ]; then
14+
# freedesktop.org and systemd
15+
. /etc/os-release
16+
platform="${NAME}"
17+
os_version="${VERSION_ID}"
18+
elif type lsb_release >/dev/null 2>&1; then
19+
# linuxbase.org
20+
platform=$(lsb_release -si)
21+
os_version=$(lsb_release -sr)
22+
elif [ -f /etc/lsb-release ]; then
23+
# For some versions of Debian/Ubuntu without lsb_release command
24+
. /etc/lsb-release
25+
platform="${DISTRIB_ID}"
26+
os_version="${DISTRIB_RELEASE}"
27+
elif [ -f /etc/debian_version ]; then
28+
# Older Debian/Ubuntu/etc.
29+
platform=Debian
30+
os_version=$(cat /etc/debian_version)
31+
elif [ -f /etc/centos-release ]; then
32+
# Older Red Hat, CentOS, etc.
33+
platform=CentOS
34+
os_version=$(cat /etc/centos-release | sed 's/.* \([0-9]\).*/\1/')
35+
else
36+
# Fall back to uname, e.g. "Linux <version>", also works for BSD, Darwin, etc.
37+
platform=$(uname -s)
38+
os_version=$(uname -r)
39+
fi
40+
}
41+
42+
function check_os_compatibility() {
43+
if [[ "${is_in_wsl}" == true && -z "${WSL_INTEROP}" ]]; then
44+
echo "GraphScope not support to run on WSL1, please use WSL2."
45+
exit 1
46+
fi
47+
48+
if [[ "${platform}" != *"Ubuntu"* ]]; then
49+
echo "This script is only available on Ubuntu"
50+
exit 1
51+
fi
52+
53+
if [[ "${platform}" == *"Ubuntu"* && "$(echo ${os_version} | sed 's/\([0-9]\)\([0-9]\).*/\1\2/')" -lt "20" ]]; then
54+
echo "This script requires Ubuntu 20 or greater."
55+
exit 1
56+
fi
57+
58+
echo "$(date '+%Y-%m-%d %H:%M:%S') preparing environment on '${platform}' '${os_version}'"
59+
}
60+
61+
function check_dependencies_version() {
62+
# python
63+
if ! hash python3; then
64+
echo "Python3 is not installed"
65+
exit 1
66+
fi
67+
ver=$(python3 -V 2>&1 | sed 's/.* \([0-9]\).\([0-9]\).*/\1\2/')
68+
if [ "$ver" -lt "36" ]; then
69+
echo "GraphScope requires python 3.6 or greater."
70+
exit 1
71+
fi
72+
}
73+
74+
function install_dependencies() {
75+
echo "$(date '+%Y-%m-%d %H:%M:%S') install dependencies."
76+
if [[ "${platform}" == *"Ubuntu"* ]]; then
77+
sudo apt-get update -y
78+
sudo apt install -y ca-certificates ccache cmake curl etcd libbrotli-dev \
79+
libbz2-dev libcurl4-openssl-dev libdouble-conversion-dev libevent-dev libgflags-dev \
80+
libboost-all-dev libgoogle-glog-dev libgrpc-dev libgrpc++-dev libgtest-dev libgsasl7-dev \
81+
libtinfo5 libkrb5-dev liblz4-dev libprotobuf-dev librdkafka-dev libre2-dev libsnappy-dev \
82+
libssl-dev libunwind-dev libutf8proc-dev libxml2-dev libz-dev libzstd-dev lsb-release maven \
83+
openjdk-8-jdk perl protobuf-compiler-grpc python3-pip uuid-dev wget zip zlib1g-dev
84+
85+
# install apache-arrow
86+
wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb -P /tmp
87+
sudo apt install -y -V /tmp/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
88+
sudo apt update
89+
sudo apt install -y libarrow-dev=1.0.1-1 libarrow-python-dev=1.0.1-1
90+
91+
# install zookeeper
92+
wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz -P /tmp
93+
tar xf /tmp/zookeeper-3.4.14.tar.gz -C /tmp/
94+
cp /tmp/zookeeper-3.4.14/conf/zoo_sample.cfg /tmp/zookeeper-3.4.14/conf/zoo.cfg
95+
sudo cp -r /tmp/zookeeper-3.4.14 /usr/local/zookeeper || true
96+
97+
# rust
98+
wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz -P /tmp
99+
sudo tar -C /usr/local -xzf /tmp/go1.15.5.linux-amd64.tar.gz
100+
curl -sf -L https://static.rust-lang.org/rustup.sh | sh -s -- -y --profile minimal --default-toolchain 1.48.0
101+
102+
# clean
103+
rm -fr /tmp/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
104+
rm -fr /tmp/zookeeper-3.4.14.tar.gz /tmp/zookeeper-3.4.14 /tmp/go1.15.5.linux-amd64.tar.gz
105+
106+
# install python packages for vineyard codegen
107+
pip3 install -U pip --user
108+
pip3 install libclang parsec setuptools wheel twine --user
109+
fi
110+
111+
check_dependencies_version
112+
}
113+
114+
get_os_version
115+
116+
check_os_compatibility
117+
118+
install_dependencies
119+
120+
echo "The script has successfully install dependencies for GraphScope."
121+
122+
set +x
123+
set +e
124+
set +o pipefail

0 commit comments

Comments
 (0)