chacha: Move ARMv8 OPENSSL_armcap_P dispatching from assembly to C.

Take a step towards removing all uses of OPENSSL_armcap_P from the ARMv8 assembly code. Change-Id: Ic1a75e107017b33f3e88b8eae503b788e37ca70a Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/64207 Reviewed-by: Bob Beck <[email protected]> Reviewed-by: David Benjamin <[email protected]> Commit-Queue: David Benjamin <[email protected]>
briansmith · Dec 19, 2023 · 5b6a9cf · 5b6a9cf
1 parent 538b2a6
commit 5b6a9cf
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 34 deletions.
diff --git a/crypto/chacha/asm/chacha-armv8.pl b/crypto/chacha/asm/chacha-armv8.pl
@@ -122,9 +122,6 @@ sub ROUND {
 $code.=<<___;
 #include <openssl/arm_arch.h>
 
-.extern	OPENSSL_armcap_P
-.hidden	OPENSSL_armcap_P
-
 .section .rodata
 
 .align	5
@@ -136,24 +133,10 @@ sub ROUND {
 
 .text
 
-.globl	ChaCha20_ctr32
-.type	ChaCha20_ctr32,%function
+.globl	ChaCha20_ctr32_nohw
+.type	ChaCha20_ctr32_nohw,%function
 .align	5
-ChaCha20_ctr32:
-	AARCH64_VALID_CALL_TARGET
-	cbz	$len,.Labort
-#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10
-	adrp	@x[0],:pg_hi21_nc:OPENSSL_armcap_P
-#else
-	adrp	@x[0],:pg_hi21:OPENSSL_armcap_P
-#endif
-	cmp	$len,#192
-	b.lo	.Lshort
-	ldr	w17,[@x[0],:lo12:OPENSSL_armcap_P]
-	tst	w17,#ARMV7_NEON
-	b.ne	ChaCha20_neon
-
-.Lshort:
+ChaCha20_ctr32_nohw:
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29,x30,[sp,#-96]!
 	add	x29,sp,#0
@@ -276,7 +259,6 @@ sub ROUND {
 	ldp	x27,x28,[x29,#80]
 	ldp	x29,x30,[sp],#96
 	AARCH64_VALIDATE_LINK_REGISTER
-.Labort:
 	ret
 
 .align	4
@@ -334,7 +316,7 @@ sub ROUND {
 	ldp	x29,x30,[sp],#96
 	AARCH64_VALIDATE_LINK_REGISTER
 	ret
-.size	ChaCha20_ctr32,.-ChaCha20_ctr32
+.size	ChaCha20_ctr32_nohw,.-ChaCha20_ctr32_nohw
 ___
 
 {{{
@@ -375,9 +357,10 @@ sub NEONROUND {
 
 $code.=<<___;
 
-.type	ChaCha20_neon,%function
+.globl	ChaCha20_ctr32_neon
+.type	ChaCha20_ctr32_neon,%function
 .align	5
-ChaCha20_neon:
+ChaCha20_ctr32_neon:
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29,x30,[sp,#-96]!
 	add	x29,sp,#0
@@ -690,7 +673,7 @@ sub NEONROUND {
 	ldp	x29,x30,[sp],#96
 	AARCH64_VALIDATE_LINK_REGISTER
 	ret
-.size	ChaCha20_neon,.-ChaCha20_neon
+.size	ChaCha20_ctr32_neon,.-ChaCha20_ctr32_neon
 ___
 {
 my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;

diff --git a/crypto/chacha/chacha.c b/crypto/chacha/chacha.c
@@ -60,7 +60,22 @@ void CRYPTO_hchacha20(uint8_t out[32], const uint8_t key[32],
   OPENSSL_memcpy(&out[16], &x[12], sizeof(uint32_t) * 4);
 }
 
-#if defined(CHACHA20_ASM)
+#if defined(CHACHA20_ASM_NOHW)
+static void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,
+                           const uint32_t key[8], const uint32_t counter[4]) {
+#if defined(CHACHA20_ASM_NEON)
+  if (ChaCha20_ctr32_neon_capable(in_len)) {
+    ChaCha20_ctr32_neon(out, in, in_len, key, counter);
+    return;
+  }
+#endif
+  if (in_len > 0) {
+    ChaCha20_ctr32_nohw(out, in, in_len, key, counter);
+  }
+}
+#endif
+
+#if defined(CHACHA20_ASM) || defined(CHACHA20_ASM_NOHW)
 
 void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len,
                       const uint8_t key[32], const uint8_t nonce[12],

diff --git a/crypto/chacha/chacha_test.cc b/crypto/chacha/chacha_test.cc
@@ -347,7 +347,25 @@ TEST(ChaChaTest, CounterOverflow) {
   }
 }
 
-#if defined(CHACHA20_ASM) && defined(SUPPORTS_ABI_TEST)
+#if defined(SUPPORTS_ABI_TEST)
+
+static void check_abi(uint8_t *out, const uint8_t *in, size_t in_len,
+                      const uint32_t key[8], const uint32_t counter[4]) {
+#if defined(CHACHA20_ASM)
+  CHECK_ABI(ChaCha20_ctr32, out, in, in_len, key, counter);
+#endif
+#if defined(CHACHA20_ASM_NEON)
+  if (ChaCha20_ctr32_neon_capable(in_len)) {
+    CHECK_ABI(ChaCha20_ctr32_neon, out, in, in_len, key, counter);
+  }
+#endif
+#if defined(CHACHA20_ASM_NOHW)
+  if (in_len > 0) {
+    CHECK_ABI(ChaCha20_ctr32_nohw, out, in, in_len, key, counter);
+  }
+#endif
+}
+
 TEST(ChaChaTest, ABI) {
   uint32_t key[8];
   OPENSSL_memcpy(key, kKey, sizeof(key));
@@ -357,14 +375,15 @@ TEST(ChaChaTest, ABI) {
   auto buf = std::make_unique<uint8_t[]>(sizeof(kInput));
   for (size_t len = 0; len <= 32; len++) {
     SCOPED_TRACE(len);
-    CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len, key, kCounterNonce);
+    check_abi(buf.get(), kInput, len, key, kCounterNonce);
   }
 
   for (size_t len : {32 * 2, 32 * 4, 32 * 8, 32 * 16, 32 * 24}) {
     SCOPED_TRACE(len);
-    CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len, key, kCounterNonce);
+    check_abi(buf.get(), kInput, len, key, kCounterNonce);
     // Cover the partial block paths.
-    CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len + 15, key, kCounterNonce);
+    check_abi(buf.get(), kInput, len + 15, key, kCounterNonce);
   }
 }
-#endif  // CHACHA20_ASM && SUPPORTS_ABI_TEST
+
+#endif  // SUPPORTS_ABI_TEST
diff --git a/crypto/chacha/internal.h b/crypto/chacha/internal.h
@@ -17,6 +17,8 @@
 
 #include <openssl/base.h>
 
+#include "../internal.h"
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -27,11 +29,24 @@ extern "C" {
 void CRYPTO_hchacha20(uint8_t out[32], const uint8_t key[32],
                       const uint8_t nonce[16]);
 
-#if !defined(OPENSSL_NO_ASM) &&                         \
-    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
-     defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
+#if !defined(OPENSSL_NO_ASM) && \
+    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || defined(OPENSSL_ARM))
+
 #define CHACHA20_ASM
 
+#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64)
+
+#define CHACHA20_ASM_NOHW
+
+#define CHACHA20_ASM_NEON
+OPENSSL_INLINE int ChaCha20_ctr32_neon_capable(size_t len) {
+  return (len >= 192) && CRYPTO_is_NEON_capable();
+}
+void ChaCha20_ctr32_neon(uint8_t *out, const uint8_t *in, size_t in_len,
+                         const uint32_t key[8], const uint32_t counter[4]);
+#endif
+
+#if defined(CHACHA20_ASM)
 // ChaCha20_ctr32 encrypts |in_len| bytes from |in| and writes the result to
 // |out|. If |in| and |out| alias, they must be equal.
 //
@@ -44,6 +59,12 @@ void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,
                     const uint32_t key[8], const uint32_t counter[4]);
 #endif
 
+#if defined(CHACHA20_ASM_NOHW)
+// ChaCha20_ctr32_nohw is like |ChaCha20_ctr32| except |in_len| must be nonzero.
+void ChaCha20_ctr32_nohw(uint8_t *out, const uint8_t *in, size_t in_len,
+                         const uint32_t key[8], const uint32_t counter[4]);
+#endif
+
 
 #if defined(__cplusplus)
 }  // extern C