From 821694376ad3653005bc2641cf39d3761dc81c24 Mon Sep 17 00:00:00 2001 From: Amir Sanjar Date: Thu, 15 Jul 2021 20:42:13 +0000 Subject: [PATCH 1/5] adding ppc64le support for kibana --- .../src/common/kibana/install_kibana.sh | 4 +++ .../common/kibana/patch0-kibana-arm64.diff | 36 ------------------- .../common/kibana/patch0-kibana-platform.diff | 26 ++++++++++++++ .../kibana/patch4-kibana-os-platform.diff | 11 ++++++ 4 files changed, 41 insertions(+), 36 deletions(-) delete mode 100644 bigtop-packages/src/common/kibana/patch0-kibana-arm64.diff create mode 100644 bigtop-packages/src/common/kibana/patch0-kibana-platform.diff create mode 100644 bigtop-packages/src/common/kibana/patch4-kibana-os-platform.diff diff --git a/bigtop-packages/src/common/kibana/install_kibana.sh b/bigtop-packages/src/common/kibana/install_kibana.sh index 89cade9901..72c56704c2 100644 --- a/bigtop-packages/src/common/kibana/install_kibana.sh +++ b/bigtop-packages/src/common/kibana/install_kibana.sh @@ -113,6 +113,10 @@ fi if [ "${ARCH}" = "aarch64" ];then cp -ar $BUILD_DIR/kibana-*-linux-arm64/* $PREFIX/$LIB_DIR/ fi +if [ "${ARCH}" = "ppc64le" ];then + cp -ar $BUILD_DIR/kibana-*-linux-ppc64*/* $PREFIX/$LIB_DIR/ +fi + chmod 755 $PREFIX/$LIB_DIR/* -R diff --git a/bigtop-packages/src/common/kibana/patch0-kibana-arm64.diff b/bigtop-packages/src/common/kibana/patch0-kibana-arm64.diff deleted file mode 100644 index 6c71b8200d..0000000000 --- a/bigtop-packages/src/common/kibana/patch0-kibana-arm64.diff +++ /dev/null @@ -1,36 +0,0 @@ -diff -ruaN kibana-5.4.1-orig/tasks/build/os_packages.js kibana-5.4.1-arm64-support/tasks/build/os_packages.js ---- kibana-5.4.1-orig/tasks/build/os_packages.js 2020-07-22 14:51:45.240239878 +0800 -+++ kibana-5.4.1-arm64-support/tasks/build/os_packages.js 2020-07-22 14:51:34.283874336 +0800 -@@ -14,7 +14,7 @@ - grunt.file.mkdir(targetDir); - - config.get('platforms') -- .filter(({ name }) => /linux-x86(_64)?$/.test(name)) -+ .filter(({ name }) => /linux-(x86|x86_64|arm64)?$/.test(name)) - .forEach(({ buildDir, debArch, rpmArch }) => { - const baseOptions = [ - '--force', -diff -ruaN kibana-5.4.1-orig/tasks/config/platforms.js kibana-5.4.1-arm64-support/tasks/config/platforms.js ---- kibana-5.4.1-orig/tasks/config/platforms.js 2020-07-22 14:51:45.244239999 +0800 -+++ kibana-5.4.1-arm64-support/tasks/config/platforms.js 2020-07-22 14:51:34.283874336 +0800 -@@ -12,6 +12,7 @@ - 'darwin-x64', - 'linux-x64', - 'linux-x86', -+ 'linux-arm64', - 'windows-x86' - ].map(function (baseName) { - const win = baseName === 'windows-x86'; -@@ -42,10 +43,12 @@ - let rpmArch; - if (name.match('linux')) { - debArch = name.match('x86_64') ? 'amd64' : 'i386'; -+ debArch = name.match('arm64') ? 'arm64' : debArch; - debName = `kibana-${version}-${debArch}.deb`; - debPath = resolve(rootPath, `target/${debName}`); - - rpmArch = name.match('x86_64') ? 'x86_64' : 'i686'; -+ rpmArch = name.match('arm64') ? 'arm64' : rpmArch; - rpmName = `kibana-${version}-${rpmArch}.rpm`; - rpmPath = resolve(rootPath, `target/${rpmName}`); - } diff --git a/bigtop-packages/src/common/kibana/patch0-kibana-platform.diff b/bigtop-packages/src/common/kibana/patch0-kibana-platform.diff new file mode 100644 index 0000000000..54c5fd186b --- /dev/null +++ b/bigtop-packages/src/common/kibana/patch0-kibana-platform.diff @@ -0,0 +1,26 @@ +--- kibana-5.4.1/tasks/config/platforms.js 2017-05-27 12:53:15.000000000 +0000 ++++ kibana-5.4.1/tasks/config/platforms.js 2021-07-14 21:37:50.423345645 +0000 +@@ -12,6 +12,8 @@ + 'darwin-x64', + 'linux-x64', + 'linux-x86', ++ 'linux-arm64', ++ 'linux-ppc64le', + 'windows-x86' + ].map(function (baseName) { + const win = baseName === 'windows-x86'; +@@ -42,10 +44,14 @@ + let rpmArch; + if (name.match('linux')) { + debArch = name.match('x86_64') ? 'amd64' : 'i386'; ++ debArch = name.match('arm64') ? 'arm64' : debArch; ++ debArch = name.match('ppc64le') ? 'ppc64le' : debArch; + debName = `kibana-${version}-${debArch}.deb`; + debPath = resolve(rootPath, `target/${debName}`); + + rpmArch = name.match('x86_64') ? 'x86_64' : 'i686'; ++ rpmArch = name.match('arm64') ? 'arm64' : rpmArch; ++ rpmArch = name.match('ppc64le') ? 'ppc64le' : rpmArch; + rpmName = `kibana-${version}-${rpmArch}.rpm`; + rpmPath = resolve(rootPath, `target/${rpmName}`); + } diff --git a/bigtop-packages/src/common/kibana/patch4-kibana-os-platform.diff b/bigtop-packages/src/common/kibana/patch4-kibana-os-platform.diff new file mode 100644 index 0000000000..aec4dec521 --- /dev/null +++ b/bigtop-packages/src/common/kibana/patch4-kibana-os-platform.diff @@ -0,0 +1,11 @@ +--- kibana-5.4.1/tasks/build/os_packages.js 2017-05-27 12:53:15.000000000 +0000 ++++ kibana-5.4.1/tasks/build/os_packages.js 2021-07-14 21:37:50.423345645 +0000 +@@ -14,7 +14,7 @@ + grunt.file.mkdir(targetDir); + + config.get('platforms') +- .filter(({ name }) => /linux-x86(_64)?$/.test(name)) ++ .filter(({ name }) => /linux-(x86|x86_64|arm64|ppc64le)?$/.test(name)) + .forEach(({ buildDir, debArch, rpmArch }) => { + const baseOptions = [ + '--force', From 5a27344ac09b37b6d685973cb37d865de48c9ce8 Mon Sep 17 00:00:00 2001 From: Amir Sanjar Date: Tue, 24 Aug 2021 16:17:22 -0400 Subject: [PATCH 2/5] fixzeppelin on ppc64le - JIRA BIGTOP-3560 --- .../src/common/zeppelin/do-component-build | 7 +++ ...-java-1.28.0-add-support-for-ppc64le.patch | 12 +++++ bigtop_toolchain/manifests/grpc.pp | 45 +++++++++++++++++++ bigtop_toolchain/manifests/installer.pp | 1 + 4 files changed, 65 insertions(+) create mode 100644 bigtop_toolchain/files/grpc-java-1.28.0-add-support-for-ppc64le.patch create mode 100644 bigtop_toolchain/manifests/grpc.pp diff --git a/bigtop-packages/src/common/zeppelin/do-component-build b/bigtop-packages/src/common/zeppelin/do-component-build index dc170b429a..b42a764f4b 100644 --- a/bigtop-packages/src/common/zeppelin/do-component-build +++ b/bigtop-packages/src/common/zeppelin/do-component-build @@ -45,6 +45,13 @@ BUILD_OPTS="-Dhadoop3.2.version=${HADOOP_VERSION} \ -Phadoop3 \ -Pbuild-distr" +if [ $HOSTTYPE = "powerpc64le" ] ; then + mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.5.0 \ + -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.5.1.1/bin/protoc + + mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.28.0 -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/src/grpc-java-1.28.0/compiler/build/exe/java_plugin/protoc-gen-grpc-java +fi + if [ "$ZEPPELIN_RUN_TESTS" = "true" ]; then BUILD_OPTS="$BUILD_OPTS -DskipTests=false" else diff --git a/bigtop_toolchain/files/grpc-java-1.28.0-add-support-for-ppc64le.patch b/bigtop_toolchain/files/grpc-java-1.28.0-add-support-for-ppc64le.patch new file mode 100644 index 0000000000..e40fd3a902 --- /dev/null +++ b/bigtop_toolchain/files/grpc-java-1.28.0-add-support-for-ppc64le.patch @@ -0,0 +1,12 @@ +diff --git a/compiler/build.gradle b/compiler/build.gradle +index 60d3a43..55acee0 100644 +--- a/compiler/build.gradle ++++ b/compiler/build.gradle +@@ -105,7 +105,6 @@ model { + // Link other (system) libraries dynamically. + // Clang under OSX doesn't support these options. + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", +- "-static-libstdc++", + "-Wl,-Bdynamic", "-lpthread", "-s" + } + addEnvArgs("LDFLAGS", linker.args) diff --git a/bigtop_toolchain/manifests/grpc.pp b/bigtop_toolchain/manifests/grpc.pp new file mode 100644 index 0000000000..7234083530 --- /dev/null +++ b/bigtop_toolchain/manifests/grpc.pp @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class bigtop_toolchain::grpc { + + require bigtop_toolchain::packages + $grpc_version = '1.28.0' + $proto_version = '3.5.1.1' + $proto_home = "/usr/local/protobuf-3.5.1.1" + + if ($architecture == 'ppc64le') { + + exec { "download grpc-java ${grpc_version}": + cwd => "/usr/src", + command => "/usr/bin/wget https://github.com/grpc/grpc-java/archive/refs/tags/v${grpc_version}.tar.gz && mkdir -p grpc-java-${grpc_version} && /bin/tar -xvzf v${grpc_version}.tar.gz -C grpc-java-${grpc_version} --strip-components=1", + creates => "/usr/src/grpc-java-${grpc_version}", + } + + file { "/usr/src/grpc-java-${grpc_version}/grpc-java-${grpc_version}-add-support-for-ppc64le.patch": + source => "puppet:///modules/bigtop_toolchain/grpc-java-${grpc_version}-add-support-for-ppc64le.patch", + require => Exec["download grpc-java ${grpc_version}"], + } + + exec { "build grpc-java ${grpc_version}": + cwd => "/usr/src/grpc-java-${grpc_version}", + command => "/usr/bin/patch -p1 < grpc-java-${grpc_version}-add-support-for-ppc64le.patch && export LDFLAGS=-L/${proto_home}/lib && export CXXFLAGS=-I/${proto_home}/include && export LD_LIBRARY_PATH=/${proto_home}/lib && cd compiler && ../gradlew java_pluginExecutable -PskipAndroid=true", + creates => "/usr/local/grpc-java-${grpc_version}", + require => File["/usr/src/grpc-java-${grpc_version}/grpc-java-${grpc_version}-add-support-for-ppc64le.patch"], + timeout => 3000 + } + } + +} diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp index 8a09490df7..c2536ea35c 100644 --- a/bigtop_toolchain/manifests/installer.pp +++ b/bigtop_toolchain/manifests/installer.pp @@ -19,6 +19,7 @@ include bigtop_toolchain::ant include bigtop_toolchain::gradle include bigtop_toolchain::protobuf + include bigtop_toolchain::grpc include bigtop_toolchain::packages include bigtop_toolchain::env include bigtop_toolchain::user From 1e750787b971443b6c626cd06091c7380785d405 Mon Sep 17 00:00:00 2001 From: afsanjar Date: Tue, 24 Aug 2021 16:28:50 -0500 Subject: [PATCH 3/5] Update .travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 05789c2570..8e3c53c515 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,3 +20,6 @@ install: script: - mvn test -B + +arch: + - ppc64le From 305f61d4820d3d87783dfba58318b8d009eff6f0 Mon Sep 17 00:00:00 2001 From: afsanjar Date: Wed, 15 Sep 2021 15:51:28 -0400 Subject: [PATCH 4/5] BIGTOP-3560 part 2 --- .../src/common/zeppelin/patch2-ppc64le.diff | 25 +++++++++++++++++++ bigtop_toolchain/manifests/grpc.pp | 4 ++- bigtop_toolchain/manifests/installer.pp | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 bigtop-packages/src/common/zeppelin/patch2-ppc64le.diff diff --git a/bigtop-packages/src/common/zeppelin/patch2-ppc64le.diff b/bigtop-packages/src/common/zeppelin/patch2-ppc64le.diff new file mode 100644 index 0000000000..fabfe33bc7 --- /dev/null +++ b/bigtop-packages/src/common/zeppelin/patch2-ppc64le.diff @@ -0,0 +1,25 @@ +diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json +index de83a629d..619a04d0c 100644 +--- a/zeppelin-web/package.json ++++ b/zeppelin-web/package.json +@@ -7,7 +7,7 @@ + }, + "scripts": { + "clean": "rimraf dist && rimraf .tmp", +- "postinstall": "bower install --silent", ++ "postinstall": "bower install --silent --allow-root", + "prebuild": "npm-run-all clean lint:once", + "build:dist": "npm-run-all prebuild && grunt pre-webpack-dist && webpack && grunt post-webpack-dist", + "build:ci": "npm-run-all prebuild && grunt pre-webpack-ci && webpack && grunt post-webpack-dist", +diff --git a/zeppelin-web/pom.xml b/zeppelin-web/pom.xml +index 54e97053d..2433addd5 100644 +--- a/zeppelin-web/pom.xml ++++ b/zeppelin-web/pom.xml +@@ -83,7 +83,7 @@ + + + ${web.e2e.enabled} +- install --no-lockfile ++ install --no-lockfile --unsafe-perm + + diff --git a/bigtop_toolchain/manifests/grpc.pp b/bigtop_toolchain/manifests/grpc.pp index 7234083530..bb7f2f8f73 100644 --- a/bigtop_toolchain/manifests/grpc.pp +++ b/bigtop_toolchain/manifests/grpc.pp @@ -15,7 +15,9 @@ class bigtop_toolchain::grpc { - require bigtop_toolchain::packages + require bigtop_toolchain::jdk + require bigtop_toolchain::protobuf + $grpc_version = '1.28.0' $proto_version = '3.5.1.1' $proto_home = "/usr/local/protobuf-3.5.1.1" diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp index c2536ea35c..bcf9ee72e0 100644 --- a/bigtop_toolchain/manifests/installer.pp +++ b/bigtop_toolchain/manifests/installer.pp @@ -19,11 +19,11 @@ include bigtop_toolchain::ant include bigtop_toolchain::gradle include bigtop_toolchain::protobuf - include bigtop_toolchain::grpc include bigtop_toolchain::packages include bigtop_toolchain::env include bigtop_toolchain::user include bigtop_toolchain::renv + include bigtop_toolchain::grpc stage { 'last': require => Stage['main'], From e7980eed2764e6ccb805b4857988e0b583897692 Mon Sep 17 00:00:00 2001 From: afsanjar Date: Sun, 19 Sep 2021 07:25:26 -0400 Subject: [PATCH 5/5] Fixing centos 7 build break based on BIGTOP-3564. --- bigtop-packages/src/common/zeppelin/do-component-build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigtop-packages/src/common/zeppelin/do-component-build b/bigtop-packages/src/common/zeppelin/do-component-build index b42a764f4b..0559af5864 100644 --- a/bigtop-packages/src/common/zeppelin/do-component-build +++ b/bigtop-packages/src/common/zeppelin/do-component-build @@ -50,6 +50,9 @@ if [ $HOSTTYPE = "powerpc64le" ] ; then -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.5.1.1/bin/protoc mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.28.0 -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/src/grpc-java-1.28.0/compiler/build/exe/java_plugin/protoc-gen-grpc-java +sed -i "s|v12.3.1|v12.22.1|" pom.xml +sed -i "s|6.9.0|6.14.12|" pom.xml + fi if [ "$ZEPPELIN_RUN_TESTS" = "true" ]; then