CI fixes (cossacklabs#1034)

Rix Rust CI builds (cossacklabs#1032)

* Pin log dependency to 0.4.17
* Pin byteorder dependency to 1.4.3

Last versions that still work with current MSRV 1.58.

Fix sanitizers CI job (cossacklabs#1033)

* Switch to GCC 10
* Install `libgcc-10-dev` that provides file `libtsan_preinit.o` needed
  for thread sanitizer

Change priority of PREFIX in CFLAGS and LDFLAGS (cossacklabs#1031)

* Change include dir priority in CFLAGS
Move 'CFLAGS += -I/usr/local/include' below engine selection macros,
so that if OS have OpenSSL header files installed in /usr/local/include,
it won't mess with vendored BoringSSL or any other selected engine.

* Put /usr/local/lib in separate LDFLAGS
Introduce ADDITIONAL_LDFLAGS macro and put it after both LDFLAGS
and CRYPTO_ENGINE_LDFLAGS during linking.

* Add optional verbose logging to Makefile
Add VERBOSE option to Makefile. If set, print executed command
(compiler, linker etc) even for successful runs, and run ldd on created
shared libraries.

* Enable verbose builds on macOS jobs

.github/workflows/test-core.yaml

@@ -36,6 +36,7 @@ jobs:
     name: Unit tests
     runs-on: ${{ matrix.os }}
     env:
+      VERBOSE: 1
       SOTER_KDF_RUN_LONG_TESTS: yes
       MATRIX_OS: ${{ matrix.os }}
     strategy:
@@ -153,20 +154,20 @@ jobs:
         run: |
           sudo sh -c 'echo "DEBIAN_FRONTEND=noninteractive" >> /etc/environment'
           sudo apt update
-          sudo apt install --yes gcc-8 clang-8 make libssl-dev
+          sudo apt install --yes gcc-10 libgcc-10-dev clang-8 make libssl-dev
       - name: Check out code
         uses: actions/checkout@v2
       # We test only OpenSSL flavor to not expand the testing matrix too much
       # (rebuilding BoringSSL is not fun and takes much time)
       - name: Check with GCC (ASan)
         if: always()
-        run: make clean test CC=gcc-8 WITH_ASAN=1
+        run: make clean test CC=gcc-10 WITH_ASAN=1
       - name: Check with GCC (TSan)
         if: always()
-        run: make clean test CC=gcc-8 WITH_TSAN=1
+        run: make clean test CC=gcc-10 WITH_TSAN=1
       - name: Check with GCC (UBSan)
         if: always()
-        run: make clean test CC=gcc-8 WITH_UBSAN=1
+        run: make clean test CC=gcc-10 WITH_UBSAN=1
       - name: Check with Clang (ASan)
         if: always()
         run: make clean test CC=clang-8 WITH_ASAN=1

Makefile

@@ -88,9 +88,6 @@ pkgconfigdir ?= $(libdir)/pkgconfig
 # Add Themis source directory to search paths
 CFLAGS  += -I$(INC_PATH) -I$(SRC_PATH) -I$(SRC_PATH)/wrappers/themis/
 LDFLAGS += -L$(BIN_PATH)
-# Not all platforms include /usr/local in default search path
-CFLAGS  += -I/usr/local/include
-LDFLAGS += -L/usr/local/lib
 # Build shared libraries
 CFLAGS  += -fPIC
 
@@ -111,8 +108,13 @@ OK_STRING=$(MOVE_COLUMN)$(OK_COLOR)[OK]$(NO_COLOR)
 ERROR_STRING=$(MOVE_COLUMN)$(ERROR_COLOR)[ERRORS]$(NO_COLOR)
 WARN_STRING=$(MOVE_COLUMN)$(WARN_COLOR)[WARNINGS]$(NO_COLOR)
 
+ifeq ($(VERBOSE),)
 PRINT_OK = printf "$@ $(OK_STRING)\n"
 PRINT_OK_ = printf "$(OK_STRING)\n"
+else
+PRINT_OK = printf "$@ $(OK_STRING)\n" && printf "$(CMD)\n"
+PRINT_OK_ = printf "$(OK_STRING)\n" && printf "$(CMD)\n"
+endif
 PRINT_ERROR = printf "$@ $(ERROR_STRING)\n" && printf "$(CMD)\n$$LOG\n" && false
 PRINT_ERROR_ = printf "$(ERROR_STRING)\n" && printf "$(CMD)\n$$LOG\n" && false
 PRINT_WARNING = printf "$@ $(WARN_STRING)\n" && printf "$(CMD)\n$$LOG\n"
@@ -166,6 +168,18 @@ ifneq ($(ENGINE_LIB_PATH),)
 	CRYPTO_ENGINE_LIB_PATH = $(ENGINE_LIB_PATH)
 endif
 
+# Basic compiler flags (lower priority than selected engine)
+# We got /usr/local as default PREFIX and not all platforms include that path in default search path.
+# Make sure whatever PREFIX is used, includes and libs are searched there.
+#
+# These two additional flags, -I and -L, need to be _after_ engine flags to not override it.
+# CFLAGS is populated with CRYPTO_ENGINE_CFLAGS few lines above, so we could add -I to CFLAGS.
+# LDFLAGS and CRYPTO_ENGINE_LDFLAGS are used separately, in this same order, so new macro was
+# introduced, ADDITIONAL_LDFLAGS, to be used after CRYPTO_ENGINE_LDFLAGS, so that LDFLAGS remain
+# at the beginning of linker flags.
+CFLAGS  += -I$(includedir)
+ADDITIONAL_LDFLAGS += -L$(libdir)
+
 ifneq ($(AUTH_SYM_ALG),)
 	CFLAGS += -D$(AUTH_SYM_ALG)
 endif

benches/rust/Cargo.toml

@@ -13,7 +13,7 @@ criterion = { version = "0.3.4", features = ["cargo_bench_support", "html_report
 csv = "~1.1"
 # Freeze `rayon` and `rayon-core` versions (first is used by `criterion`)
 # so that benchmarks still build/run with Rust 1.58.
-# FIXME: remove thiese two after we bump minimum required Rust version
+# FIXME: remove these two after we bump minimum required Rust version
 rayon = "=1.6.1"
 rayon-core = "=1.10.1"
 

benches/themis/Cargo.toml

@@ -14,7 +14,7 @@ criterion = { version = "0.3.4", features = ["cargo_bench_support", "html_report
 csv = "~1.1"
 # Freeze `rayon` and `rayon-core` versions (first is used by `criterion`)
 # so that benchmarks still build/run with Rust 1.58.
-# FIXME: remove thiese two after we bump minimum required Rust version
+# FIXME: remove these two after we bump minimum required Rust version
 rayon = "=1.6.1"
 rayon-core = "=1.10.1"
 

jni/themis_jni.mk

@@ -53,7 +53,7 @@ endif
 
 $(OBJ_PATH)/jni/%: CFLAGS += $(jvm_includes)
 
-$(BIN_PATH)/$(LIBTHEMISJNI_SO): CMD = $(CC) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) -lthemis
+$(BIN_PATH)/$(LIBTHEMISJNI_SO): CMD = $(CC) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) -lthemis $(ADDITIONAL_LDFLAGS)
 
 $(BIN_PATH)/$(LIBTHEMISJNI_SO): $(THEMIS_JNI_OBJ) $(BIN_PATH)/$(LIBTHEMIS_SO)
 	@mkdir -p $(@D)

src/soter/soter.mk

@@ -78,15 +78,28 @@ $(BIN_PATH)/$(LIBSOTER_A): $(SOTER_OBJ) $(SOTER_ENGINE_DEPS)
 	@echo -n "link "
 	@$(BUILD_CMD)
 
-$(BIN_PATH)/$(LIBSOTER_SO): CMD = $(CC) -shared -o $@ $(filter %.o %a, $^) $(LDFLAGS) $(CRYPTO_ENGINE_LDFLAGS) $(LIBSOTER_SO_LDFLAGS)
+$(BIN_PATH)/$(LIBSOTER_SO): CMD = $(CC) -shared -o $@ $(filter %.o %a, $^) $(LDFLAGS) $(CRYPTO_ENGINE_LDFLAGS) $(ADDITIONAL_LDFLAGS) $(LIBSOTER_SO_LDFLAGS)
 
 $(BIN_PATH)/$(LIBSOTER_SO): $(SOTER_OBJ) $(SOTER_ENGINE_DEPS)
 	@mkdir -p $(@D)
+ifneq ($(VERBOSE),)
+	@echo "LDFLAGS=$(LDFLAGS)"
+	@echo "CRYPTO_ENGINE_LDFLAGS=$(CRYPTO_ENGINE_LDFLAGS)"
+	@echo "ADDITIONAL_LDFLAGS=$(ADDITIONAL_LDFLAGS)"
+	@echo "LIBSOTER_SO_LDFLAGS=$(LIBSOTER_SO_LDFLAGS)"
+endif
 	@echo -n "link "
 	@$(BUILD_CMD)
 ifneq ($(LIBSOTER_SO),$(LIBSOTER_LINK))
 	@ln -sf $(LIBSOTER_SO) $(BIN_PATH)/$(LIBSOTER_LINK)
 endif
+ifneq ($(VERBOSE),)
+ifdef IS_MACOS
+	-otool -L "$@"
+else
+	-ldd "$@"
+endif
+endif
 
 $(BIN_PATH)/libsoter.pc:
 	@mkdir -p $(BIN_PATH)

src/themis/themis.mk

@@ -62,7 +62,7 @@ $(BIN_PATH)/$(LIBTHEMIS_A): $(THEMIS_OBJ)
 	@echo -n "link "
 	@$(BUILD_CMD)
 
-$(BIN_PATH)/$(LIBTHEMIS_SO): CMD = $(CC) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) -lsoter $(LIBTHEMIS_SO_LDFLAGS)
+$(BIN_PATH)/$(LIBTHEMIS_SO): CMD = $(CC) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) -lsoter $(ADDITIONAL_LDFLAGS) $(LIBTHEMIS_SO_LDFLAGS)
 
 $(BIN_PATH)/$(LIBTHEMIS_SO): $(BIN_PATH)/$(LIBSOTER_SO) $(THEMIS_OBJ)
 	@mkdir -p $(@D)
@@ -71,6 +71,13 @@ $(BIN_PATH)/$(LIBTHEMIS_SO): $(BIN_PATH)/$(LIBSOTER_SO) $(THEMIS_OBJ)
 ifneq ($(LIBTHEMIS_SO),$(LIBTHEMIS_LINK))
 	@ln -sf $(LIBTHEMIS_SO) $(BIN_PATH)/$(LIBTHEMIS_LINK)
 endif
+ifneq ($(VERBOSE),)
+ifdef IS_MACOS
+	-otool -L "$@"
+else
+	-ldd "$@"
+endif
+endif
 
 $(BIN_PATH)/libthemis.pc:
 	@mkdir -p $(BIN_PATH)

src/wrappers/themis/rust/Cargo.toml

@@ -30,8 +30,10 @@ zeroize = "1"
 
 [dev-dependencies]
 base64 = "0.10.0"
-byteorder = "1.2.7"
+# Freeze `log` and `byteorder` so that tests still build/run with Rust 1.58.
+# FIXME: remove/update strict version requirement after we bump minimum required Rust version
+byteorder = "=1.4.3"
 clap = "2.32"
 lazy_static = "1.2.0"
-log = "0.4.6"
+log = "=0.4.17"
 env_logger = "0.6.0"

tests/soter/soter.mk

@@ -65,7 +65,7 @@ endif
 $(SOTER_TEST_BIN): $(BIN_PATH)/$(LIBSOTER_SO)
 endif
 
-$(SOTER_TEST_BIN): CMD = $(CC) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(SOTER_TEST_LDFLAGS)
+$(SOTER_TEST_BIN): CMD = $(CC) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(ADDITIONAL_LDFLAGS) $(SOTER_TEST_LDFLAGS)
 
 $(SOTER_TEST_BIN): $(SOTER_TEST_OBJ) $(COMMON_TEST_OBJ)
 	@mkdir -p $(@D)

tests/themis/themis.mk

@@ -47,7 +47,7 @@ endif
 $(THEMIS_TEST_BIN): $(BIN_PATH)/$(LIBTHEMIS_SO)
 endif
 
-$(THEMIS_TEST_BIN): CMD = $(CC) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(THEMIS_TEST_LDFLAGS)
+$(THEMIS_TEST_BIN): CMD = $(CC) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(ADDITIONAL_LDFLAGS) $(THEMIS_TEST_LDFLAGS)
 
 $(THEMIS_TEST_BIN): $(THEMIS_TEST_OBJ) $(COMMON_TEST_OBJ)
 	@mkdir -p $(@D)

tests/themispp/themispp.mk

@@ -39,7 +39,7 @@ endif
 $(TEST_BIN_PATH)/themispp_test: $(BIN_PATH)/$(LIBTHEMIS_SO)
 endif
 
-$(TEST_BIN_PATH)/themispp_test: CMD = $(CXX) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(THEMISPP_TEST_LDFLAGS)
+$(TEST_BIN_PATH)/themispp_test: CMD = $(CXX) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) $(ADDITIONAL_LDFLAGS) $(THEMISPP_TEST_LDFLAGS)
 
 $(TEST_BIN_PATH)/themispp_test: $(THEMISPP_TEST_OBJ) $(COMMON_TEST_OBJ)
 	@echo -n "link "

tools/afl/fuzzy.mk

@@ -38,7 +38,7 @@ FUZZ_UTILS = $(filter-out $(addsuffix .o,$(FUZZ_TOOLS)),$(FUZZ_OBJS))
 
 # Build sources with access to fuzzing headers and link tools to $(FUZZ_THEMIS_LIB).
 $(FUZZ_OBJS): CFLAGS += -I$(FUZZ_SRC_PATH)
-$(FUZZ_TOOLS): LDFLAGS += $(FUZZ_THEMIS_LIB) $(FUZZ_SOTER_LIB) $(CRYPTO_ENGINE_LDFLAGS)
+$(FUZZ_TOOLS): LDFLAGS += $(FUZZ_THEMIS_LIB) $(FUZZ_SOTER_LIB) $(CRYPTO_ENGINE_LDFLAGS) $(ADDITIONAL_LDFLAGS)
 
 # afl-clang is partially configured via environment variables. For one, it likes to
 # talk on stdout so tell it to pipe down a bit. Additionally, address sanitizer builds