core: Deny building with OpenSSL 3.0

.github/workflows/test-core.yaml

@@ -46,7 +46,7 @@ jobs:
         run: |
           if [[ "$(uname)" = "Darwin" ]]
           then
-            brew install cmake ninja [email protected]
+            brew install cmake ninja [email protected] openssl@3
           else
             sudo sh -c 'echo "DEBIAN_FRONTEND=noninteractive" >> /etc/environment'
             sudo apt update
@@ -74,6 +74,19 @@ jobs:
       - name: Run test suite (WITH_SCELL_COMPAT)
         if: always()
         run: make test BUILD_PATH=build-compat
+      - name: Ensure OpenSSL 3.0 fails (macOS only)
+        if: ${{ matrix.os == 'macos-latest' }}
+        run: |
+          # Themis uses OpenSSL 1.1 by default if installed.
+          # Explicitly request OpenSSL 3.0 by pointing the build into OpenSSL 3.0's paths.
+          openssl3=$(brew --prefix openssl@3)
+          if ! make ENGINE=openssl BUILD_PATH=build-openssl-3.0 ENGINE_INCLUDE_PATH=$openssl3/include ENGINE_LIB_PATH=$openssl3/lib
+          then
+            true
+          else
+            echo "Build with OpenSSL 3.0 did not fail when it should have"
+            exit 1
+          fi
 
   examples:
     name: Code examples

CHANGELOG.md

@@ -68,6 +68,7 @@ _Code:_
   - Fixed multiple buffer overflows in Secure Message ([#763](https://github.com/cossacklabs/themis/pull/763)).
   - Fixed cross-compilation on macOS by setting `ARCH` and `SDK` variables ([#849](https://github.com/cossacklabs/themis/pull/849)).
   - Updated embedded BoringSSL to the latest version ([#812](https://github.com/cossacklabs/themis/pull/812)).
+  - Builds with OpenSSL 3.0 will result in a compilation error for the time being ([#872](https://github.com/cossacklabs/themis/pull/872)).
 
 - **Android**
 
@@ -180,6 +181,7 @@ _Infrastructure:_
 - Embedded BoringSSL now builds faster if Ninja is available ([#837](https://github.com/cossacklabs/themis/pull/837)).
 - Embedded BoringSSL can now be cross-compiled on macOS by setting `ARCH` and `SDK` variables ([#849](https://github.com/cossacklabs/themis/pull/849)).
 - Builds on macOS use OpenSSL 1.1 from Homebrew by default ([#871](https://github.com/cossacklabs/themis/pull/871)).
+- Builds with OpenSSL 3.0 are currently **not supported** ([#872](https://github.com/cossacklabs/themis/pull/872)).
 
 
 ## [0.13.12](https://github.com/cossacklabs/themis/releases/tag/0.13.12), July 26th 2021

Makefile

@@ -186,6 +186,10 @@ ifeq ($(RSA_KEY_LENGTH),8192)
 	CFLAGS += -DTHEMIS_RSA_KEY_LENGTH=RSA_KEY_LENGTH_8192
 endif
 
+ifeq ($(WITH_EXPERIMENTAL_OPENSSL_3_SUPPORT),yes)
+	CFLAGS += -DTHEMIS_EXPERIMENTAL_OPENSSL_3_SUPPORT=1
+endif
+
 ########################################################################
 #
 # Compilation flags for C/C++ code

src/soter/openssl/soter_engine.h

@@ -23,6 +23,15 @@
 
 #include "soter/soter_asym_sign.h"
 
+/*
+ * For the time being Themis and Soter do not support OpenSSL 3.0.
+ * The code seems to build fine but it fails the tests, so we're not sure
+ * that it is safe to use Soter with OpenSSL 3.0.
+ */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !THEMIS_EXPERIMENTAL_OPENSSL_3_SUPPORT
+#error OpenSSL 3.0 is currently not supported
+#endif
+
 struct soter_hash_ctx_type {
     EVP_MD_CTX* evp_md_ctx;
 };