Skip to content

Commit 09f8f3c

Browse files
GUIGUI
committed
Add Ubuntu 16.04 packaging. Fix and improve some packaging things.
The packaging makefile is now more flexible with specific targets for building a single distro's packages. The download packages script is also improved to only download the necessary packages. Also fix an erroneous --rpm-dist flag when building .deb packages. We also need a newer version of librdkafka for Ubuntu 16.04 compatibility, but the newer v0.9.1 seems to compile fine against rsyslog.
1 parent 1990d92 commit 09f8f3c

File tree

9 files changed

+139
-60
lines changed

9 files changed

+139
-60
lines changed

‎build/cmake/rsyslog.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ ExternalProject_Add(
3535
)
3636

3737
if(ENABLE_HADOOP_ANALYTICS)
38+
# There's a small dependency on Python for librdkafka's Makefile:
39+
# https://github.com/edenhill/librdkafka/blob/0.9.1/Makefile#L8
40+
find_package(PythonInterp REQUIRED)
41+
3842
ExternalProject_Add(
3943
librdkafka
4044
URL https://github.com/edenhill/librdkafka/archive/${LIBRDKAFKA_VERSION}.tar.gz
@@ -45,6 +49,7 @@ if(ENABLE_HADOOP_ANALYTICS)
4549
)
4650
endif()
4751

52+
list(APPEND RSYSLOG_DEPENDS json-c)
4853
list(APPEND RSYSLOG_DEPENDS libestr)
4954
if(ENABLE_HADOOP_ANALYTICS)
5055
list(APPEND RSYSLOG_DEPENDS librdkafka)

‎build/cmake/versions.cmake‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,8 @@ set(LIBESTR_VERSION 0.1.10)
2020
set(LIBESTR_HASH bd655e126e750edd18544b88eb1568d200a424a0c23f665eb14bbece07ac703c)
2121
set(LIBGEOIP_VERSION 1.6.9)
2222
set(LIBGEOIP_HASH 7475942dc8155046dddb4846f587a7e6)
23-
# Hold at 0.8 - 0.9 doesn't seem to be compatible with rsyslog.
24-
set(LIBRDKAFKA_VERSION 0.8.6)
25-
set(LIBRDKAFKA_HASH 1b77543f9be82d3f700c0ef98f494990)
23+
set(LIBRDKAFKA_VERSION 0.9.1)
24+
set(LIBRDKAFKA_HASH feb25faed02815f60ff363b2f40ba1b9)
2625
set(LUAROCKS_VERSION 2.3.0)
2726
set(LUAROCKS_HASH a38126684cf42b7d0e7a3c7cf485defb)
2827
set(LUAROCK_ARGPARSE_VERSION 0.5.0-1)

‎build/package/Makefile‎

Lines changed: 65 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,74 +4,112 @@ LOG_DIR:=$(WORK_DIR)/log
44

55
.PHONY: \
66
docker \
7+
docker_build \
8+
docker_verify \
79
docker_centos6 \
8-
docker_centos6_verify_only \
10+
docker_centos6_build \
11+
docker_centos6_verify \
912
docker_centos7 \
10-
docker_centos7_verify_only \
13+
docker_centos7_build \
14+
docker_centos7_verify \
1115
docker_ubuntu1204 \
12-
docker_ubuntu1204_verify_only \
16+
docker_ubuntu1204_build \
17+
docker_ubuntu1204_verify \
1318
docker_ubuntu1404 \
14-
docker_ubuntu1404_verify_only \
19+
docker_ubuntu1404_build \
20+
docker_ubuntu1404_verify \
21+
docker_ubuntu1604 \
22+
docker_ubuntu1604_build \
23+
docker_ubuntu1604_verify \
1524
docker_debian7 \
16-
docker_debian7_verify_only \
25+
docker_debian7_build \
26+
docker_debian7_verify \
1727
docker_debian8 \
18-
docker_debian8_verify_only \
28+
docker_debian8_build \
29+
docker_debian8_verify \
30+
docker_all
1931

2032
$(LOG_DIR):
2133
mkdir -p $@
2234

23-
$(WORK_DIR)/download_previous_packages.stamp: $(SOURCE_DIR)/build/package/verify/download_previous_packages
24-
$(SOURCE_DIR)/build/package/verify/download_previous_packages
25-
mkdir -p $(WORK_DIR)
26-
touch $@
35+
docker: | $(LOG_DIR)
36+
$(MAKE) docker_build
37+
$(MAKE) docker_verify
2738

28-
docker: $(LOG_DIR)
39+
docker_build: | $(LOG_DIR)
2940
$(SOURCE_DIR)/build/package/docker_run > $(LOG_DIR)/$(DIST).log 2>&1
30-
$(SOURCE_DIR)/build/package/verify/docker_run >> $(LOG_DIR)/$(DIST).log 2>&1
3141

32-
docker_verify_only: $(WORK_DIR)/download_previous_packages.stamp | $(LOG_DIR)
42+
docker_verify: | $(LOG_DIR)
43+
$(SOURCE_DIR)/build/package/verify/download_previous_packages >> $(LOG_DIR)/$(DIST).log 2>&1
3344
$(SOURCE_DIR)/build/package/verify/docker_run >> $(LOG_DIR)/$(DIST).log 2>&1
3445

3546
docker_centos6:
3647
DIST=centos-6 $(MAKE) docker
3748

38-
docker_centos6_verify_only:
39-
DIST=centos-6 $(MAKE) docker_verify_only
49+
docker_centos6_build:
50+
DIST=centos-6 $(MAKE) docker_build
51+
52+
docker_centos6_verify:
53+
DIST=centos-6 $(MAKE) docker_verify
4054

4155
docker_centos7:
4256
DIST=centos-7 $(MAKE) docker
4357

44-
docker_centos7_verify_only:
45-
DIST=centos-7 $(MAKE) docker_verify_only
58+
docker_centos7_build:
59+
DIST=centos-7 $(MAKE) docker_build
60+
61+
docker_centos7_verify:
62+
DIST=centos-7 $(MAKE) docker_verify
4663

4764
docker_ubuntu1204:
4865
DIST=ubuntu-12.04 $(MAKE) docker
4966

50-
docker_ubuntu1204_verify_only:
51-
DIST=ubuntu-12.04 $(MAKE) docker_verify_only
67+
docker_ubuntu1204_build:
68+
DIST=ubuntu-12.04 $(MAKE) docker_build
69+
70+
docker_ubuntu1204_verify:
71+
DIST=ubuntu-12.04 $(MAKE) docker_verify
5272

5373
docker_ubuntu1404:
5474
DIST=ubuntu-14.04 $(MAKE) docker
5575

56-
docker_ubuntu1404_verify_only:
57-
DIST=ubuntu-14.04 $(MAKE) docker_verify_only
76+
docker_ubuntu1404_build:
77+
DIST=ubuntu-14.04 $(MAKE) docker_build
78+
79+
docker_ubuntu1404_verify:
80+
DIST=ubuntu-14.04 $(MAKE) docker_verify
81+
82+
docker_ubuntu1604:
83+
DIST=ubuntu-16.04 $(MAKE) docker
84+
85+
docker_ubuntu1604_build:
86+
DIST=ubuntu-16.04 $(MAKE) docker_build
87+
88+
docker_ubuntu1604_verify:
89+
DIST=ubuntu-16.04 $(MAKE) docker_verify
5890

5991
docker_debian7:
6092
DIST=debian-7 $(MAKE) docker
6193

62-
docker_debian7_verify_only:
63-
DIST=debian-7 $(MAKE) docker_verify_only
94+
docker_debian7_build:
95+
DIST=debian-7 $(MAKE) docker_build
96+
97+
docker_debian7_verify:
98+
DIST=debian-7 $(MAKE) docker_verify
6499

65100
docker_debian8:
66101
DIST=debian-8 $(MAKE) docker
67102

68-
docker_debian8_verify_only:
69-
DIST=debian-8 $(MAKE) docker_verify_only
103+
docker_debian8_build:
104+
DIST=debian-8 $(MAKE) docker_build
70105

71-
docker_all: \
72-
docker_centos6 \
106+
docker_debian8_verify:
107+
DIST=debian-8 $(MAKE) docker_verify
108+
109+
docker_all: docker_centos6 \
73110
docker_centos7 \
74111
docker_ubuntu1204 \
75112
docker_ubuntu1404 \
113+
docker_ubuntu1604 \
76114
docker_debian7 \
77115
docker_debian8

‎build/package/build_package‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ else
2222
fi
2323

2424
fpm_args=()
25-
fpm_args+=("--rpm-dist" "$rpm_dist")
2625
fpm_args+=("-t" "$package_type")
2726
fpm_args+=("-s" "dir")
2827
fpm_args+=("--verbose")

‎build/package/verify/download_previous_packages‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ set -e -u
88

99
source_dir="$(dirname "$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")")"
1010

11+
function compare_version {
12+
echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
13+
}
14+
1115
previous_versions=(
1216
0.8.0-1
1317
0.9.0-1
@@ -25,6 +29,10 @@ distros=(
2529
debian-8
2630
)
2731

32+
if [ -n "${DIST:-}" ]; then
33+
distros=($DIST)
34+
fi
35+
2836
sourceforge_url_root="http://pilotfiber.dl.sourceforge.net/project/api-umbrella"
2937
bintray_url_root="https://bintray.com/artifact/download/nrel"
3038
checksums="$(cat "$source_dir/sha256sums.txt")"
@@ -38,56 +46,62 @@ for dist in "${distros[@]}"; do
3846
case "$dist" in
3947
centos-6)
4048
filename="api-umbrella-${version}.el6.x86_64.rpm"
49+
url_prefix="$bintray_url_root/api-umbrella-el6"
4150
if [ "$version" == "0.8.0-1" ]; then
4251
url_prefix="$sourceforge_url_root/el/6"
43-
else
44-
url_prefix="$bintray_url_root/api-umbrella-el6"
4552
fi
4653
;;
4754
centos-7)
4855
filename="api-umbrella-${version}.el7.x86_64.rpm"
56+
url_prefix="$bintray_url_root/api-umbrella-el7"
4957
if [ "$version" == "0.8.0-1" ]; then
5058
url_prefix="$sourceforge_url_root/el/7"
51-
else
52-
url_prefix="$bintray_url_root/api-umbrella-el7"
5359
fi
5460
;;
5561
ubuntu-12.04)
5662
filename="api-umbrella_${version}~precise_amd64.deb"
63+
url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
5764
if [ "$version" == "0.8.0-1" ]; then
5865
url_filename="api-umbrella_${version}_amd64.deb"
5966
url_prefix="$sourceforge_url_root/ubuntu/12.04"
60-
else
61-
url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
6267
fi
6368
;;
6469
ubuntu-14.04)
6570
filename="api-umbrella_${version}~trusty_amd64.deb"
71+
url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
6672
if [ "$version" == "0.8.0-1" ]; then
6773
url_filename="api-umbrella_${version}_amd64.deb"
6874
url_prefix="$sourceforge_url_root/ubuntu/14.04"
69-
else
70-
url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
75+
fi
76+
;;
77+
ubuntu-16.04)
78+
filename="api-umbrella_${version}~xenial_amd64.deb"
79+
url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
80+
if [ "$(compare_version "$version")" -lt "$(compare_version "0.12.0")" ]; then
81+
# No Ubuntu 16.04 packages until v0.12
82+
filename=""
7183
fi
7284
;;
7385
debian-7)
7486
filename="api-umbrella_${version}~wheezy_amd64.deb"
87+
url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
7588
if [ "$version" == "0.8.0-1" ]; then
7689
url_filename="api-umbrella_${version}_amd64.deb"
7790
url_prefix="$sourceforge_url_root/debian/7"
78-
else
79-
url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
8091
fi
8192
;;
8293
debian-8)
8394
filename="api-umbrella_${version}~jessie_amd64.deb"
84-
if [ "$version" == "0.8.0-1" ]; then
85-
# No Debian 8 packages for API Umbrella v0.8.
95+
url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
96+
if [ "$(compare_version "$version")" -lt "$(compare_version "0.9.0")" ]; then
97+
# No Debian 8 packages until v0.9
8698
filename=""
87-
else
88-
url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
8999
fi
90100
;;
101+
*)
102+
echo "Unknown distribution: $dist"
103+
exit 1
104+
;;
91105
esac
92106

93107
if [ -n "$filename" ]; then

‎build/package/verify/spec/localhost/service_spec.rb‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,21 @@
100100
end
101101

102102
RSpec.shared_examples("package upgrade") do |package_version|
103+
# Skip testing upgrades if we don't have binary packages for certain distro
104+
# and version combinations.
105+
case(ENV["DIST"])
106+
when "debian-8"
107+
# No Debian 8 packages until v0.9
108+
if(Gem::Version.new(package_version) < Gem::Version.new("0.9.0-1"))
109+
next
110+
end
111+
when "ubuntu-16.04"
112+
# No Ubuntu 16.04 packages until v0.12
113+
if(Gem::Version.new(package_version) < Gem::Version.new("0.12.0-1"))
114+
next
115+
end
116+
end
117+
103118
def ensure_uninstalled
104119
command_result = command("/etc/init.d/api-umbrella stop")
105120
command_result.exit_status
@@ -520,12 +535,7 @@ def install_package(version)
520535
end
521536
end
522537

523-
# We don't have Debian 8 builds of API Umbrella v0.8, so skip testing that
524-
# upgrade path for Debian 8.
525-
if(ENV["DIST"] != "debian-8")
526-
it_behaves_like "package upgrade", "0.8.0-1"
527-
end
528-
538+
it_behaves_like "package upgrade", "0.8.0-1"
529539
it_behaves_like "package upgrade", "0.9.0-1"
530540
it_behaves_like "package upgrade", "0.10.0-1"
531541
it_behaves_like "package upgrade", "0.11.0-1"

‎build/package_dependencies.sh‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/usr/bin/env bash
22

3+
set -e -u
4+
35
if [ -f /etc/redhat-release ]; then
46
core_package_dependencies=(
57
# General
@@ -54,6 +56,8 @@ if [ -f /etc/redhat-release ]; then
5456
openssl-devel
5557
patch
5658
pcre-devel
59+
pkgconfig
60+
python
5761
rpm-build
5862
rsync
5963
tar
@@ -65,10 +69,22 @@ if [ -f /etc/redhat-release ]; then
6569
java-1.8.0-openjdk-devel
6670
)
6771
elif [ -f /etc/debian_version ]; then
72+
libffi_version=6
73+
openjdk_version=7
74+
75+
# shellcheck disable=SC1091
76+
source /etc/os-release
77+
if [[ "$ID" == "debian" && "$VERSION_ID" == "7" ]]; then
78+
libffi_version=5
79+
elif [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]]; then
80+
openjdk_version=8
81+
fi
82+
6883
core_package_dependencies=(
6984
# General
7085
bash
7186
libc6
87+
libffi$libffi_version
7288
libncurses5
7389
libpcre3
7490
libuuid1
@@ -81,7 +97,7 @@ elif [ -f /etc/debian_version ]; then
8197
tcl
8298

8399
# ElasticSearch
84-
openjdk-7-jre-headless
100+
openjdk-$openjdk_version-jre-headless
85101

86102
# init.d script helpers
87103
sysvinit-utils
@@ -91,7 +107,7 @@ elif [ -f /etc/debian_version ]; then
91107
procps
92108
)
93109
hadoop_analytics_package_dependencies=(
94-
openjdk-7-jre-headless
110+
openjdk-$openjdk_version-jre-headless
95111
)
96112
core_build_dependencies=(
97113
autoconf
@@ -114,6 +130,8 @@ elif [ -f /etc/debian_version ]; then
114130
make
115131
openssl
116132
patch
133+
pkg-config
134+
python
117135
rsync
118136
tar
119137
tcl-dev
@@ -122,13 +140,11 @@ elif [ -f /etc/debian_version ]; then
122140
xz-utils
123141
)
124142
hadoop_analytics_build_dependencies=(
125-
openjdk-7-jdk-headless
143+
openjdk-$openjdk_version-jdk
126144
)
127145

128-
if lsb_release --codename --short | grep wheezy; then
129-
core_package_dependencies+=("libffi5")
130-
else
131-
core_package_dependencies+=("libffi6")
146+
if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]]; then
147+
core_build_dependencies+=("libtool-bin")
132148
fi
133149
else
134150
echo "Unknown build system"

0 commit comments

Comments
 (0)