From 3d5c400eef16e3df5dde088343e0e8ac8e19a694 Mon Sep 17 00:00:00 2001 From: Alexei Lozovsky Date: Mon, 13 Jul 2020 20:31:46 +0300 Subject: [PATCH 1/5] Include embedded BoringSSL into libsoter.a Currently, when building Themis with embedded BoringSSL (the one from git submodule), the shared library is linked against the BoringSSL static library and it is embedded into the dylib. That way resulting Themis dylib does not depend on the system OpenSSL. Instead it uses the BoringSSL which we build and embed into it. However, right now Themis static library does not include embedded BoringSSL in it. The user is expected to find "libcrypto.a" and "libdecrepit.a" in BoringSSL build directory, distribute them along with "libthemis.a" and "libsoter.a", and include all four libraries when building the application. Why is this an issue? --------------------- This is not particularly convenient since BoringSSL libraries are buried in the BoringSSL build directory, not available in the usual Themis build directory. It is quite possible that developers will forget about them entirely. They are also not included into the packages we build. Furthermore, Soter is built with and expects that particular version of BoringSSL to be linked into the application. While BoringSSL generally does a good job at maintaining ABI compatibility, the user may accidentally link some OpenSSL from their system instead of BoringSSL that was built with Themis. This may break in a subtle way. How we resolve it ----------------- In order to avoid all those issues, let's embed BoringSSL into Soter static library, like we do with the shared library. That way the users will have to link only against "libthemis.a" and "libsoter.a". Note that this is actual only for the case when we are building Themis with embedded BoringSSL. That is, the BoringSSL we provide in submodule. We should not embed system OpenSSL, or any BoringSSL built and provided by the user. In that case the users are expected to take care of the cryptography provider library themselves. On portability -------------- Unfortunately, one does not simply merge static libraries. The usual "ar" tool cannot do that: if you mention *.o files and *.a files it will simply include *.a files into the archive as is and it will confuse the linker which unpacks only one layer of static libraries. The other traditional tool which is intended for this use case is "libtool". On macOS is has somewhat sane interface and is available on the default system installation. But on Linux systems the GNU libtool is typically used. It has... well... a little bit wacky interface and it is normally used with Autotools. The problem with Autotools is that Themis build system is not expected to bow to their idiosyncrasies. Furthermore, on all distros we support libtool is not available in the default installation. I do not really want to add a new dependency -- **especially** dependency on Autotools -- just for that one use case. So instead of that, here's an utility script which uses "ar" to merge multiple static libraries. It has issues of its own, but at least it's portable and should work on any UNIX-like system out of the box. On symbol conficts ------------------ As a final note, embedding BoringSSL into the static library has some consequences. Because of the way static linkage and static libraries work, this means that "libsoter.a" provides BoringSSL symbols which may conflict with and/or shadow other BoringSSL or OpenSSL symbols. This changeset does not do anything about it. Like before, this is the issue that the users have to solve themselves. Though, we can still do something about it later if we manage to use BoringSSL symbol prefixes. --- CHANGELOG.md | 4 +++ scripts/merge-static-libs.sh | 66 ++++++++++++++++++++++++++++++++++++ src/soter/soter.mk | 11 ++++-- 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100755 scripts/merge-static-libs.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b04b0989..815ace31f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Changes that are currently in development and have not been released yet. _Code:_ +- **Core** + + - Include embedded BoringSSL into Soter for convenience ([#681](https://github.com/cossacklabs/themis/pull/681)). + - **Android** - AndroidThemis is now available on JCenter ([#679](https://github.com/cossacklabs/themis/pull/679)). diff --git a/scripts/merge-static-libs.sh b/scripts/merge-static-libs.sh new file mode 100755 index 000000000..75283debd --- /dev/null +++ b/scripts/merge-static-libs.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Merge multiple static libraries into one. +# +# scripts/merge-static-libs.sh libtarget.a [libdep1.a ...] +# +# Libraries libdep1.a are merged into libtarget.a. That is, all object files +# from those libraries are copied into the target library. +# +# Note that no deduplication is performed so use this script only once. + +set -euo pipefail + +AR=${AR:-ar} + +help() { + cat $0 | awk 'NR == 3, /^$/ { print substr($0, 3) }' +} + +while [[ $# -gt 0 ]] +do + case "$1" in + -h|--help) help; exit;; + --) shift; break;; + -*) + echo >&2 "Unknown option: $1" + echo >&2 + help + exit 1 + ;; + *) + break + ;; + esac +done + +if [[ $# -lt 1 ]] +then + echo >&2 "Invalid command-line: missing target library name" + echo >&2 + help + exit 1 +fi + +target="$1" +shift + +tempdir="$(mktemp -d)" +trap 'rm -rf "$tempdir"' EXIT + +while [[ $# -gt 0 ]] +do + # Unfortunately, ar can extract files only into the current directory + # and there is no easy and portable way to convert possibly relative path + # into an absolute one. So... this is slow but it works. I'm sorry. + cp "$1" "$tempdir/library.a" + mkdir "$tempdir/contents" + cd "$tempdir/contents" + "$AR" x ../library.a + cd "$OLDPWD" + # Actually merge the contents of the library into the target one. + find "$tempdir/contents" -type f -name '*.o' -print0 | xargs -0 "$AR" rs "$target" + # Clean up and go on. + rm -rf "$tempdir/library.a" "$tempdir/contents" + shift +done diff --git a/src/soter/soter.mk b/src/soter/soter.mk index 852899efd..ebdf3dabb 100644 --- a/src/soter/soter.mk +++ b/src/soter/soter.mk @@ -59,9 +59,14 @@ SOTER_STATIC = $(BIN_PATH)/$(LIBSOTER_A) $(SOTER_ENGINE_DEPS) $(SOTER_OBJ): CFLAGS += -DSOTER_EXPORT -$(BIN_PATH)/$(LIBSOTER_A): CMD = $(AR) rcs $@ $(filter %.o, $^) - -$(BIN_PATH)/$(LIBSOTER_A): $(SOTER_OBJ) +# First build Soter library, then merge embedded crypto engine libs into it. +# On macOS this may cause warnings about files with no symbols in BoringSSL, +# suppress those warnings with some Bash wizardry. +$(BIN_PATH)/$(LIBSOTER_A): CMD = $(AR) rcs $@ $(filter %.o, $^) \ + && scripts/merge-static-libs.sh $@ $(filter %.a, $^) \ + $(if $(IS_MACOS),> >(grep -v 'has no symbols$$')) + +$(BIN_PATH)/$(LIBSOTER_A): $(SOTER_OBJ) $(SOTER_ENGINE_DEPS) @mkdir -p $(@D) @echo -n "link " @$(BUILD_CMD) From 467d674ac7c11d49c33ad45337acd70c6302f912 Mon Sep 17 00:00:00 2001 From: Alexei Lozovsky Date: Tue, 21 Jul 2020 12:03:49 +0300 Subject: [PATCH 2/5] Use Bash from PATH Instead of the usual "#!/bin/bash" shebang, use "#!/usr/bin/env bash" so that the script is executed with whatever Bash is in PATH. This makes the script a bit more flexible. --- scripts/merge-static-libs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/merge-static-libs.sh b/scripts/merge-static-libs.sh index 75283debd..eebd558e2 100755 --- a/scripts/merge-static-libs.sh +++ b/scripts/merge-static-libs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Merge multiple static libraries into one. # From 63c78bcb7bd270a19fe26b6b869ea76033468265 Mon Sep 17 00:00:00 2001 From: Alexei Lozovsky Date: Tue, 21 Jul 2020 12:08:32 +0300 Subject: [PATCH 3/5] Avoid copying library for extraction @shad has found a nice hack to convert a path into an absolute one. Use it to avoid copying libraries around for "ar" which can extract files only into the current directory. Co-authored-by: Dmytro Shapovalov --- scripts/merge-static-libs.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/merge-static-libs.sh b/scripts/merge-static-libs.sh index eebd558e2..e724b3aaa 100755 --- a/scripts/merge-static-libs.sh +++ b/scripts/merge-static-libs.sh @@ -50,17 +50,19 @@ trap 'rm -rf "$tempdir"' EXIT while [[ $# -gt 0 ]] do - # Unfortunately, ar can extract files only into the current directory - # and there is no easy and portable way to convert possibly relative path - # into an absolute one. So... this is slow but it works. I'm sorry. - cp "$1" "$tempdir/library.a" - mkdir "$tempdir/contents" - cd "$tempdir/contents" - "$AR" x ../library.a + if [[ ! -f "$1" ]] + then + echo >&2 "No such file: $1" + exit 1 + fi + # Cast a possibly relative path to the absolute one. + src="$(cd "$(dirname "$1")" >/dev/null 2>&1 && pwd)/$(basename "$1")" + cd "$tempdir" + "$AR" x "$src" cd "$OLDPWD" # Actually merge the contents of the library into the target one. - find "$tempdir/contents" -type f -name '*.o' -print0 | xargs -0 "$AR" rs "$target" + find "$tempdir" -type f -name '*.o' -print0 | xargs -0 "$AR" rs "$target" # Clean up and go on. - rm -rf "$tempdir/library.a" "$tempdir/contents" + rm -rf "$tempdir"/* shift done From c241a4c0a80ce9b92cb50d322d63f03e55f0674d Mon Sep 17 00:00:00 2001 From: Alexei Lozovsky Date: Tue, 21 Jul 2020 12:18:20 +0300 Subject: [PATCH 4/5] Add "scripts" directory to libthemis-src libthemis-src crate needs full source code of Themis as in performs builds. This includes all build dependencies, such as newly added script for merging static libraries. Add a symlink to the script directory to include it into libthemis-src distribution. --- src/wrappers/themis/rust/libthemis-src/themis/scripts | 1 + 1 file changed, 1 insertion(+) create mode 120000 src/wrappers/themis/rust/libthemis-src/themis/scripts diff --git a/src/wrappers/themis/rust/libthemis-src/themis/scripts b/src/wrappers/themis/rust/libthemis-src/themis/scripts new file mode 120000 index 000000000..b56e1a9f4 --- /dev/null +++ b/src/wrappers/themis/rust/libthemis-src/themis/scripts @@ -0,0 +1 @@ +../../../../../../scripts \ No newline at end of file From 10a2e9f763bf1d328b97385edb3a4d9febc204f9 Mon Sep 17 00:00:00 2001 From: Alexei Lozovsky Date: Tue, 21 Jul 2020 12:50:12 +0300 Subject: [PATCH 5/5] Avoid duplicate linkage of embedded BoringSSL Since now we include BoringSSL into "libsoter.a" if needed, we don't have to mention in Soter dependencies for static linkage anymore. This is particularly important for WasmThemis. --- src/soter/soter.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/soter/soter.mk b/src/soter/soter.mk index ebdf3dabb..0524f3135 100644 --- a/src/soter/soter.mk +++ b/src/soter/soter.mk @@ -55,7 +55,7 @@ SOTER_AUD = $(patsubst $(SRC_PATH)/%,$(AUD_PATH)/%, $(SOTER_AUD_SRC)) FMT_FIXUP += $(patsubst %,$(OBJ_PATH)/%.fmt_fixup, $(SOTER_FMT_SRC)) FMT_CHECK += $(patsubst %,$(OBJ_PATH)/%.fmt_check, $(SOTER_FMT_SRC)) -SOTER_STATIC = $(BIN_PATH)/$(LIBSOTER_A) $(SOTER_ENGINE_DEPS) +SOTER_STATIC = $(BIN_PATH)/$(LIBSOTER_A) $(SOTER_OBJ): CFLAGS += -DSOTER_EXPORT