Simplify Android build and bring up to date

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/boringssl/src"]
+	path = third_party/boringssl/src
+	url = https://boringssl.googlesource.com/boringssl

build.gradle

@@ -1,20 +1,23 @@
-import org.apache.tools.ant.taskdefs.condition.Os
+apply plugin: 'com.android.library'
 
 buildscript {
     repositories {
-        mavenCentral()
+        google()
+        jcenter()
     }
 
     dependencies {
-        classpath 'com.android.tools.build:gradle:1.1.3'
+        classpath 'com.android.tools.build:gradle:3.0.1'
     }
 }
 
-apply plugin: 'com.android.library'
+dependencies {
+    implementation project(":boringssl")
+}
 
 android {
-    compileSdkVersion 22
-    buildToolsVersion "22.0.1"
+    compileSdkVersion 27
+    buildToolsVersion "27.0.3"
 
     defaultConfig {
     	minSdkVersion 16
@@ -33,37 +36,12 @@ android {
 
 	}
 
-    task ndkBuild(type: Exec) {
-	workingDir 'jni'
-        commandLine getNdkBuildCmd() 
-	if (project.hasProperty('boringssl_path')){
-	    args = ["BORINGSSL_PATH=$boringssl_path"]
-	}
-    }
-
-    tasks.withType(JavaCompile) {
-        compileTask -> compileTask.dependsOn ndkBuild
-    }
+    // ensure we execute boringssl tasks first
+    tasks.whenTaskAdded({Task task -> task.dependsOn('boringssl:' + task.name)})
 
-    task cleanNative(type: Exec) {
-		workingDir 'jni'
-        commandLine getNdkBuildCmd(), 'clean'
-    }
-
-    clean.dependsOn cleanNative
-}
-
-def getNdkDir() {
-    if (System.env.ANDROID_NDK_ROOT != null)
-        return System.env.ANDROID_NDK_ROOT
-
-    throw new GradleException("NDK location not found. Define location with an ANDROID_NDK_ROOT environment variable.")
-}
-
-def getNdkBuildCmd() {
-    def ndkbuild = getNdkDir() + "/ndk-build"
-    if (Os.isFamily(Os.FAMILY_WINDOWS))
-        ndkbuild += ".cmd"
-
-    return ndkbuild
+	externalNativeBuild {
+	    ndkBuild {
+	        path "jni/Android.mk"
+	    }
+	}
 }

circle.yml

@@ -1,4 +1,6 @@
 machine:
+  java:
+    version: openjdk8
   environment:
     ANDROID_NDK_ROOT: $ANDROID_NDK
     GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
@@ -7,7 +9,7 @@ machine:
     PATH: $GOROOT/bin:$PATH
     VALGRIND_BUILD_PATH: $HOME/valgrind
     # to avoid OOM killer (https://circleci.com/docs/1.0/oom/#out-of-memory-errors-in-android-builds)
-    GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError"'
+    GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError"'
     # add define that turn off one nist test (tests/soter/soter_rand_test.c:190) that always fail on ci machine but ok on real machine
     CFLAGS: "-DCIRICLE_TEST"
     BORINGSSL_PATH: "$HOME/boringssl"
@@ -29,8 +31,13 @@ dependencies:
     # link from http://valgrind.org/downloads/current.html
     # don't fetch if was cached
     - if [ ! -d $VALGRIND_BUILD_PATH ]; then wget ftp://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2 && tar -xjf valgrind-3.13.0.tar.bz2 && cd valgrind-3.13.0 && ./configure --prefix=$VALGRIND_BUILD_PATH && make && sudo make install; fi
+    # install Android SDK packages
+    - echo y | android update sdk --no-ui --all --filter 'tool'
+    - yes | $ANDROID_HOME/tools/bin/sdkmanager 'tools' 'platform-tools' 'build-tools;27.0.3' 'platforms;android-27' 'ndk-bundle' 'system-images;android-22;default;armeabi-v7a'
+    - $ANDROID_HOME/tools/bin/avdmanager create avd --name nexus --device "Nexus 5" --package 'system-images;android-22;default;armeabi-v7a'
 
   override:
+    - git submodule update --init
     - make
     - make JAVA_HOME=/usr/lib/jvm/default-java themis_jni
     - sudo make install
@@ -39,15 +46,21 @@ dependencies:
     - sudo make rubythemis_install
     - sudo make phpthemis_install
     - if [ ! -d $BORINGSSL_PATH ]; then cd $HOME && git clone https://boringssl.googlesource.com/boringssl && cd boringssl && git checkout chromium-stable && mkdir build && cd build && cmake .. && make && cp decrepit/libdecrepit.a crypto/; fi
-    - if [ ! -d $BORINGSSL_PATH/build-armeabi-v7a ]; then cd $BORINGSSL_PATH && mkdir build-armeabi-v7a && cd build-armeabi-v7a && cmake -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
-    - if [ ! -d $BORINGSSL_PATH/build-arm64-v8a ]; then cd $BORINGSSL_PATH && mkdir build-arm64-v8a && cd build-arm64-v8a && cmake -DANDROID_ABI=arm64-v8a -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
-    - if [ ! -d $BORINGSSL_PATH/build-x86 ]; then cd $BORINGSSL_PATH && mkdir build-x86 && cd build-x86 && cmake -DANDROID_ABI=x86 -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
     - make ENGINE=boringssl ENGINE_INCLUDE_PATH=$HOME/boringssl/include ENGINE_LIB_PATH=$HOME/boringssl/build/crypto BUILD_PATH=build_with_boringssl prepare_tests_basic
     - make BUILD_PATH=cover_build COVERAGE=y prepare_tests_basic
     - make prepare_tests_all
     - mkdir -p $HOME/gopath/src/$GOTHEMIS_IMPORT
     - rsync -auv gothemis/ $HOME/gopath/src/$GOTHEMIS_IMPORT/
     - lcov --directory . --zerocounters
+
+compile:
+  override:
+    # limit CMake/Ninja build concurrency when building BoringSSL
+    # otherwise we hit the 4GB memory limit for the build container
+    - echo 'set_property(GLOBAL APPEND PROPERTY JOB_POOLS circleci_job_pool=4)' >> third_party/boringssl/src/CMakeLists.txt
+    - sed -i 's/"-GNinja"/"-DCMAKE_JOB_POOL_COMPILE=circleci_job_pool", "-GNinja"/g' third_party/boringssl/build.gradle
+    - ./gradlew --no-daemon --no-parallel --max-workers=2 build
+
 ## Customize test commands
 test:
   override:
@@ -73,10 +86,11 @@ test:
     - ${VALGRIND_BUILD_PATH}/bin/valgrind build_with_boringssl/tests/soter_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
     - ${VALGRIND_BUILD_PATH}/bin/valgrind build_with_boringssl/tests/themis_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
     # - tests/check_ios_test.sh `tests/start_ios_test.sh`
-    # start Android emulator
-    - emulator -avd circleci-android22 -no-audio -no-window:
+    # start Android emulator (we use explicit path, because the update above
+    # installs new emulator version, which overrides Circle CIs one )
+    - $ANDROID_HOME/emulator/emulator -avd nexus -noaudio -no-window -gpu off -verbose -qemu:
         background: true
     # wait for it to have booted
     - circle-android wait-for-boot
     # run Android tests
-    - ./gradlew --info -Pboringssl_path=$HOME/boringssl connectedAndroidTest
+    - ./gradlew --no-daemon --no-parallel --max-workers=2 connectedAndroidTest

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Mon Jun 01 10:43:23 EEST 2015
+#Wed Dec 27 12:10:05 UTC 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip

jni/Android.mk

@@ -14,12 +14,14 @@
 # limitations under the License.
 #
 
-LOCAL_PATH := $(call my-dir)
+BUILD_TYPE := release
 
-ifeq ($(BORINGSSL_PATH),)
-    $(error "boringSSL path must be set")
+ifeq ($(NDK_DEBUG),1)
+  BUILD_TYPE := debug
 endif
 
+LOCAL_PATH := $(call my-dir)
+
 include $(CLEAR_VARS)
 
 LOCAL_MODULE := libsoter
@@ -28,10 +30,10 @@ LOCAL_SRC_FILES := $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/*.c
 LOCAL_SRC_FILES += $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/boringssl/*.c))
 LOCAL_SRC_FILES += $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/ed25519/*.c))
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
 LOCAL_EXPORT_CFLAGS := -DBORINGSSL
-LOCAL_STATIC_LIBRARIES := libcrypto libdecrepit
+LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 
 include $(BUILD_STATIC_LIBRARY)
 
@@ -41,8 +43,8 @@ LOCAL_MODULE := libthemis
 
 LOCAL_SRC_FILES := $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/themis/*.c))
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src 
-LOCAL_STATIC_LIBRARIES := libsoter libcrypto libdecrepit
+LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 
 include $(BUILD_STATIC_LIBRARY)
 
@@ -54,8 +56,8 @@ LOCAL_SRC_FILES := themis_jni.c themis_message.c themis_keygen.c themis_cell.c t
 LOCAL_SRC_FILES += themis_compare.c
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
 LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 LOCAL_STATIC_LIBRARIES := libthemis libsoter libcrypto libdecrepit
-# LOCAL_LDFLAGS := -llog
 
 include $(BUILD_SHARED_LIBRARY)
 
@@ -85,26 +87,24 @@ LOCAL_SHARED_LIBRARIES := libthemis_jni
 
 LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src $(LOCAL_PATH)/../tests
 
-
 include $(BUILD_EXECUTABLE)
 
-# this is needed to disable targets not present in the NDK otherwise error happens
-#BUILD_HOST_EXECUTABLE := 
-#BUILD_HOST_STATIC_LIBRARY := 
-
-#include $(LOCAL_PATH)/external/openssl/Android.mk
-
-# We are making a separate target with slightly modified script to support building libcrypto with NDK
-# We put it here to make minimal changes in original Google scripts to make importing new versions easier
 include $(CLEAR_VARS)
 
-#LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := libcrypto
-LOCAL_SRC_FILES := $(BORINGSSL_PATH)/build-$(TARGET_ARCH_ABI)/crypto/libcrypto.a
-LOCAL_EXPORT_C_INCLUDES := $(BORINGSSL_PATH)/include
-include $(PREBUILT_STATIC_LIBRARY)
+LOCAL_SRC_FILES := $(LOCAL_PATH)/../third_party/boringssl/.externalNativeBuild/cmake/$(BUILD_TYPE)/$(TARGET_ARCH_ABI)/crypto/libcrypto.a
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/../third_party/boringssl/src/include
+
+ifneq (,$(wildcard $(LOCAL_SRC_FILES)))
+  include $(PREBUILT_STATIC_LIBRARY)
+endif
+
+include $(CLEAR_VARS)
 
 LOCAL_MODULE := libdecrepit
-LOCAL_SRC_FILES := $(BORINGSSL_PATH)/build-$(TARGET_ARCH_ABI)/decrepit/libdecrepit.a
-include $(PREBUILT_STATIC_LIBRARY)
+LOCAL_SRC_FILES := $(LOCAL_PATH)/../third_party/boringssl/.externalNativeBuild/cmake/$(BUILD_TYPE)/$(TARGET_ARCH_ABI)/decrepit/libdecrepit.a
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/../third_party/boringssl/src/include
 
+ifneq (,$(wildcard $(LOCAL_SRC_FILES)))
+  include $(PREBUILT_STATIC_LIBRARY)
+endif

jni/Application.mk

@@ -14,6 +14,11 @@
 # limitations under the License.
 #
 
-APP_PLATFORM := android-16
+APP_PLATFORM := android-21
 APP_MODULES := libthemis_jni
-APP_ABI := armeabi-v7a arm64-v8a x86
+# we needs this to prevent ndk-build complaining about missing
+# prebuilt boringssl static libraries
+APP_ALLOW_MISSING_DEPS := true
+# currently boringssl does not build well with clang for all archs
+# so we use GCC for Themis as well for maximum compatibility
+NDK_TOOLCHAIN_VERSION := 4.9

settings.gradle

@@ -0,0 +1,2 @@
+include ':boringssl'
+project(':boringssl').projectDir = new File('third_party/boringssl')

third_party/boringssl/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.cossacklabs.themis.boringssl" android:versionCode="1" android:versionName="0.9.6">
+</manifest>

third_party/boringssl/build.gradle

@@ -0,0 +1,43 @@
+buildscript {
+    repositories {
+        google()
+        jcenter()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.0.1'
+    }
+}
+
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 27
+    buildToolsVersion "27.0.3"
+
+    defaultConfig {
+        minSdkVersion 16
+        targetSdkVersion 16
+        externalNativeBuild {
+		    cmake {
+		        arguments "-DCMAKE_TOOLCHAIN_FILE=" + android.ndkDirectory + "/build/cmake/android.toolchain.cmake",
+		                  "-DANDROID_NATIVE_API_LEVEL=21",
+		                  "-DANDROID_TOOLCHAIN=gcc",
+		                  "-DCMAKE_BUILD_TYPE=Release",
+		                  "-GNinja"
+		    }
+	    }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path "src/CMakeLists.txt"
+        }
+    }
+
+	sourceSets {
+        main {
+            manifest.srcFile 'AndroidManifest.xml'
+        }
+	}
+}