Synchronize DH_check() mostly with OpenSSL 1.1.1 with some

simplifications and readability tweaks.  This ensures in
particular that dh->q is suitable if present.

Based on work by Stephen Henson and Bernd Edlinger in OpenSSL.

Issues with the current implementation found via regression
tests in py-cryptography.

ok inoguchi jsing

src/lib/libcrypto/dh/dh_check.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.18 2021/11/29 19:41:02 tb Exp $ */
+/* $OpenBSD: dh_check.c,v 1.19 2021/11/29 19:47:47 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young ([email protected])
  * All rights reserved.
  *
@@ -61,6 +61,8 @@
 #include <openssl/bn.h>
 #include <openssl/dh.h>
 
+#include "bn_lcl.h"
+
 int
 DH_check_params(const DH *dh, int *flags)
 {
@@ -104,65 +106,78 @@ DH_check_params(const DH *dh, int *flags)
 }
 
 /*
- * Check that p is a safe prime and
- * if g is 2, 3 or 5, check that it is a suitable generator
- * where
- * for 2, p mod 24 == 11
- * for 3, p mod 12 == 5
- * for 5, p mod 10 == 3 or 7
- * should hold.
+ * Check that p is a safe prime and that g is a suitable generator.
  */
 
 int
-DH_check(const DH *dh, int *ret)
+DH_check(const DH *dh, int *flags)
 {
-	int is_prime, ok = 0;
 	BN_CTX *ctx = NULL;
-	BN_ULONG l;
-	BIGNUM *q = NULL;
+	int is_prime;
+	int ok = 0;
+
+	*flags = 0;
+
+	if (!DH_check_params(dh, flags))
+		goto err;
 
-	*ret = 0;
 	ctx = BN_CTX_new();
 	if (ctx == NULL)
 		goto err;
-	q = BN_new();
-	if (q == NULL)
-		goto err;
+	BN_CTX_start(ctx);
 
-	if (BN_is_word(dh->g, DH_GENERATOR_2)) {
-		l = BN_mod_word(dh->p, 24);
-		if (l == (BN_ULONG)-1)
+	if (dh->q != NULL) {
+		BIGNUM *quotient, *residue;
+
+		if ((quotient = BN_CTX_get(ctx)) == NULL)
+			goto err;
+		if ((residue = BN_CTX_get(ctx)) == NULL)
+			goto err;
+		if ((*flags & DH_NOT_SUITABLE_GENERATOR) == 0) {
+			/* Check g^q == 1 mod p */
+			if (!BN_mod_exp_ct(residue, dh->g, dh->q, dh->p, ctx))
+				goto err;
+			if (!BN_is_one(residue))
+				*flags |= DH_NOT_SUITABLE_GENERATOR;
+		}
+		is_prime = BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL);
+		if (is_prime < 0)
 			goto err;
-		if (l != 11)
-			*ret |= DH_NOT_SUITABLE_GENERATOR;
-	} else if (BN_is_word(dh->g, DH_GENERATOR_5)) {
-		l = BN_mod_word(dh->p, 10);
-		if (l == (BN_ULONG)-1)
+		if (is_prime == 0)
+			*flags |= DH_CHECK_Q_NOT_PRIME;
+		/* Check p == 1 mod q, i.e., q divides p - 1 */
+		if (!BN_div_ct(quotient, residue, dh->p, dh->q, ctx))
 			goto err;
-		if (l != 3 && l != 7)
-			*ret |= DH_NOT_SUITABLE_GENERATOR;
-	} else
-		*ret |= DH_UNABLE_TO_CHECK_GENERATOR;
+		if (!BN_is_one(residue))
+			*flags |= DH_CHECK_INVALID_Q_VALUE;
+		if (dh->j != NULL && BN_cmp(dh->j, quotient) != 0)
+			*flags |= DH_CHECK_INVALID_J_VALUE;
+	}
 
 	is_prime = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
 	if (is_prime < 0)
 		goto err;
 	if (is_prime == 0)
-		*ret |= DH_CHECK_P_NOT_PRIME;
-	else {
+		*flags |= DH_CHECK_P_NOT_PRIME;
+	else if (dh->q == NULL) {
+		BIGNUM *q;
+
+		if ((q = BN_CTX_get(ctx)) == NULL)
+			goto err;
 		if (!BN_rshift1(q, dh->p))
 			goto err;
 		is_prime = BN_is_prime_ex(q, BN_prime_checks, ctx, NULL);
 		if (is_prime < 0)
 			goto err;
 		if (is_prime == 0)
-			*ret |= DH_CHECK_P_NOT_SAFE_PRIME;
+			*flags |= DH_CHECK_P_NOT_SAFE_PRIME;
 	}
+
 	ok = 1;
 
  err:
+	BN_CTX_end(ctx);
 	BN_CTX_free(ctx);
-	BN_free(q);
 	return ok;
 }