Test against all 100 KAT values (#1560)

Run full KAT tests weekly. All should pass except Falcon-1024, which is pending upstream changes. --------- Co-authored-by: Michael Baentsch <[email protected]>
open-quantum-safe · Jan 30, 2024 · 7c56bcb · 7c56bcb
1 parent f606d3d
commit 7c56bcb
Show file tree

Hide file tree

Showing 16 changed files with 629 additions and 252 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -172,14 +172,14 @@ jobs:
                            -e PYTEST_ARGS="<< parameters.PYTEST_ARGS >>" \
                            -v `pwd`:/root/project \
                            openquantumsafe/ci-ubuntu-focal-arm64:latest bash \
-                           -c "cd /root/project && \
+                           -c 'cd /root/project && \
                                uname -a && \
                                mkdir build && cd build && source ~/.bashrc && \
                                cmake -GNinja -DOQS_STRICT_WARNINGS=ON $CMAKE_ARGS .. && cmake -LA .. && ninja && \
                                cd .. && mkdir -p tmp && \
                                python3 -m pytest --verbose \
                                                  --ignore=tests/test_code_conventions.py \
-                                                 --junitxml=build/test-results/pytest/test-results.xml $PYTEST_ARGS"
+                                                 --junitxml=build/test-results/pytest/test-results.xml $PYTEST_ARGS'
       - store_test_results: # Note that this command will fail when running CircleCI locally, that is expected behaviour
           path: build/test-results
       - store_artifacts:
@@ -353,43 +353,47 @@ workflows:
           context: openquantumsafe
           CONTAINER: openquantumsafe/ci-ubuntu-focal-x86_64:latest
           CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-8 -DOQS_USE_OPENSSL=OFF
-          PYTEST_ARGS: --ignore=tests/test_leaks.py
+          PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
       - linux_oqs:
           <<: *require_buildcheck
           name: ubuntu-focal-shared-noopenssl
           context: openquantumsafe
           CONTAINER: openquantumsafe/ci-ubuntu-focal-x86_64:latest
           CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-7 -DOQS_DIST_BUILD=OFF -DOQS_USE_OPENSSL=OFF -DBUILD_SHARED_LIBS=ON
-          PYTEST_ARGS: --ignore=tests/test_namespace.py --ignore=tests/test_leaks.py --numprocesses=auto
+          PYTEST_ARGS: --ignore=tests/test_namespace.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py --numprocesses=auto
       - linux_oqs:
           <<: *require_buildcheck
           name: ubuntu-focal-clang15
           context: openquantumsafe
           CONTAINER: openquantumsafe/ci-ubuntu-focal-x86_64:latest
           CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DCMAKE_C_COMPILER=clang-15 -DOQS_OPT_TARGET=skylake
+          PYTEST_ARGS: --ignore=tests/test_kat_all.py
       - linux_oqs:
           <<: *require_buildcheck
           name: ubuntu-bionic-i386
           context: openquantumsafe
           CONTAINER: openquantumsafe/ci-ubuntu-bionic-i386:latest
           CMAKE_ARGS: -DCMAKE_TOOLCHAIN_FILE=../.CMake/toolchain_x86.cmake
-          PYTEST_ARGS: --ignore=tests/test_leaks.py
+          PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
       - arm_machine:
           <<: *require_buildcheck
           name: arm64
-          PYTEST_ARGS: --numprocesses=auto --maxprocesses=10
+          PYTEST_ARGS: --numprocesses=auto --maxprocesses=10 --ignore=tests/test_kat_all.py
       - macOS:
           <<: *require_buildcheck
           name: macOS-gcc11
           CMAKE_ARGS: -DCMAKE_C_COMPILER=gcc-11
+          PYTEST_ARGS: --ignore=tests/test_kat_all.py
       - macOS:
           <<: *require_buildcheck
           name: macOS-noopenssl
           CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=OFF
+          PYTEST_ARGS: --ignore=tests/test_kat_all.py
       - macOS:
           <<: *require_buildcheck
           name: macOS-shared
           CMAKE_ARGS: -DBUILD_SHARED_LIBS=ON -DOQS_DIST_BUILD=OFF -DOQS_ENABLE_KEM_CLASSIC_MCELIECE=OFF
+          PYTEST_ARGS: --ignore=tests/test_kat_all.py
 
   commit-to-main:
     when:

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -72,27 +72,27 @@ jobs:
           - name: alpine
             container: openquantumsafe/ci-alpine-amd64:latest
             CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py
+            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
           - name: alpine-openssl-all
             container: openquantumsafe/ci-alpine-amd64:latest
             CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DOQS_USE_AES_OPENSSL=ON -DOQS_USE_SHA2_OPENSSL=ON -DOQS_USE_SHA3_OPENSSL=ON
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py
+            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
           - name: alpine-noopenssl
             container: openquantumsafe/ci-alpine-amd64:latest
             CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_USE_OPENSSL=OFF
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py
+            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
           - name: focal-nistr4-openssl
             container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
             CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=NIST_R4
-            PYTEST_ARGS: --ignore=tests/test_leaks.py
+            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
           - name: jammy-std-openssl3
             container: openquantumsafe/ci-ubuntu-jammy:latest
             CMAKE_ARGS: -DOQS_STRICT_WARNINGS=ON -DOQS_ALGS_ENABLED=STD -DBUILD_SHARED_LIBS=ON
-            PYTEST_ARGS: --ignore=tests/test_leaks.py
+            PYTEST_ARGS: --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py
           - name: address-sanitizer
             container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
             CMAKE_ARGS: -DCMAKE_C_COMPILER=clang-9 -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address
-            PYTEST_ARGS: --ignore=tests/test_distbuild.py --ignore=tests/test_leaks.py --numprocesses=auto --maxprocesses=10
+            PYTEST_ARGS: --ignore=tests/test_distbuild.py --ignore=tests/test_leaks.py --ignore=tests/test_kat_all.py --numprocesses=auto --maxprocesses=10
     container:
       image: ${{ matrix.container }}
     steps:
@@ -127,7 +127,7 @@ jobs:
           - name: armhf
             ARCH: armhf
             CMAKE_ARGS: -DOQS_ENABLE_SIG_SPHINCS=OFF -DOQS_USE_OPENSSL=OFF -DOQS_OPT_TARGET=generic
-            PYTEST_ARGS: --ignore=tests/test_alg_info.py
+            PYTEST_ARGS: --ignore=tests/test_alg_info.py --ignore=tests/test_kat_all.py
           # no longer supporting armel
           # - name: armel
           #   ARCH: armel

diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
@@ -36,3 +36,30 @@ jobs:
         timeout-minutes: 360
         run: mkdir -p tmp && SKIP_ALGS='${{ matrix.SKIP_ALGS }}' python3 -m pytest --verbose ${{ matrix.PYTEST_ARGS }}
 
+  nistkat-x64:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: generic
+            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
+            CMAKE_ARGS: -DOQS_DIST_BUILD=OFF -DOQS_OPT_TARGET=generic
+            PYTEST_ARGS: --numprocesses=auto -k 'test_kat_all'
+          - name: extensions
+            container: openquantumsafe/ci-ubuntu-focal-x86_64:latest
+            CMAKE_ARGS: -DOQS_DIST_BUILD=OFF -DOQS_OPT_TARGET=haswell
+            PYTEST_ARGS: --numprocesses=auto -k 'test_kat_all'
+    container:
+      image: ${{ matrix.container }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Configure
+        run: mkdir build && cd build && cmake -GNinja ${{ matrix.CMAKE_ARGS }} .. && cmake -LA ..
+      - name: Build
+        run: ninja
+        working-directory: build
+      - name: Run tests
+        timeout-minutes: 360
+        run: mkdir -p tmp && SKIP_ALGS='${{ matrix.SKIP_ALGS }}' python3 -m pytest --verbose ${{ matrix.PYTEST_ARGS }}
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -29,4 +29,4 @@ jobs:
         run: pip.exe install pytest pytest-xdist pyyaml
       - name: Run tests
         run: |
-          python -m pytest --numprocesses=auto -vv --maxfail=10 --ignore=tests/test_code_conventions.py --junitxml=build\test-results\pytest\test-results.xml
+          python -m pytest --numprocesses=auto -vv --maxfail=10 --ignore=tests/test_code_conventions.py --ignore=tests/test_kat_all.py --junitxml=build\test-results\pytest\test-results.xml
diff --git a/scripts/copy_from_upstream/copy_from_upstream.py b/scripts/copy_from_upstream/copy_from_upstream.py
@@ -544,20 +544,20 @@ def process_families(instructions, basedir, with_kat, with_generator):
             if with_kat:
                 if family in instructions['kems']:
                     try:
-                        if kats['kem'][scheme['pretty_name_full']] != scheme['metadata']['nistkat-sha256']:
+                        if kats['kem'][scheme['pretty_name_full']]['single'] != scheme['metadata']['nistkat-sha256']:
                             print("Info: Updating KAT for %s" % (scheme['pretty_name_full']))
                     except KeyError:  # new key
                         print("Adding new KAT for %s" % (scheme['pretty_name_full']))
                         pass
-                    kats['kem'][scheme['pretty_name_full']] = scheme['metadata']['nistkat-sha256']
+                    kats['kem'][scheme['pretty_name_full']]['single'] = scheme['metadata']['nistkat-sha256']
                 else:
                     try:
-                        if kats['sig'][scheme['pretty_name_full']] != scheme['metadata']['nistkat-sha256']:
+                        if kats['sig'][scheme['pretty_name_full']]['single'] != scheme['metadata']['nistkat-sha256']:
                             print("Info: Updating KAT for %s" % (scheme['pretty_name_full']))
                     except KeyError:  # new key
                         print("Adding new KAT for %s" % (scheme['pretty_name_full']))
                         pass
-                    kats['sig'][scheme['pretty_name_full']] = scheme['metadata']['nistkat-sha256']
+                    kats['sig'][scheme['pretty_name_full']]['single'] = scheme['metadata']['nistkat-sha256']
 
         if with_generator:
             generator(

diff --git a/src/common/rand/rand_nist.c b/src/common/rand/rand_nist.c
@@ -11,14 +11,15 @@ NIST-developed software is expressly provided "AS IS." NIST MAKES NO WARRANTY OF
 You are solely responsible for determining the appropriateness of using and distributing the software and you assume all risks associated with its use, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and the unavailability or interruption of operation. This software is not intended to be used in any situation where a failure could cause risk of injury or damage to property. The software developed by NIST employees is not subject to copyright protection within the United States.
 */
 //  SPDX-License-Identifier: Unknown
-//  Modified for liboqs by Douglas Stebila
+//  Modified for liboqs by Douglas Stebila and Spencer Wilson
 //
 
 #include <assert.h>
 #include <string.h>
 
 #include <oqs/common.h>
 #include <oqs/rand.h>
+#include <oqs/rand_nist.h>
 
 #ifdef OQS_USE_OPENSSL
 #include <openssl/conf.h>
@@ -31,13 +32,7 @@ You are solely responsible for determining the appropriateness of using and dist
 
 void OQS_randombytes_nist_kat(unsigned char *x, size_t xlen);
 
-typedef struct {
-	unsigned char Key[32];
-	unsigned char V[16];
-	int reseed_counter;
-} AES256_CTR_DRBG_struct;
-
-static AES256_CTR_DRBG_struct DRBG_ctx;
+static OQS_NIST_DRBG_struct DRBG_ctx;
 static void AES256_CTR_DRBG_Update(unsigned char *provided_data, unsigned char *Key, unsigned char *V);
 
 #ifdef OQS_USE_OPENSSL
@@ -85,7 +80,7 @@ static void AES256_ECB(unsigned char *key, unsigned char *ctr, unsigned char *bu
 #endif
 }
 
-OQS_API void OQS_randombytes_nist_kat_init_256bit(const uint8_t *entropy_input, const uint8_t *personalization_string) {
+void OQS_randombytes_nist_kat_init_256bit(const uint8_t *entropy_input, const uint8_t *personalization_string) {
 	unsigned char seed_material[48];
 
 	memcpy(seed_material, entropy_input, 48);
@@ -127,6 +122,24 @@ void OQS_randombytes_nist_kat(unsigned char *x, size_t xlen) {
 	DRBG_ctx.reseed_counter++;
 }
 
+void OQS_randombytes_nist_kat_get_state(void *out) {
+	OQS_NIST_DRBG_struct *out_state = (OQS_NIST_DRBG_struct *)out;
+	if (out_state != NULL) {
+		memcpy(out_state->Key, DRBG_ctx.Key, sizeof(DRBG_ctx.Key));
+		memcpy(out_state->V, DRBG_ctx.V, sizeof(DRBG_ctx.V));
+		out_state->reseed_counter = DRBG_ctx.reseed_counter;
+	}
+}
+
+void OQS_randombytes_nist_kat_set_state(const void *in) {
+	const OQS_NIST_DRBG_struct *in_state = (const OQS_NIST_DRBG_struct *)in;
+	if (in_state != NULL) {
+		memcpy(DRBG_ctx.Key, in_state->Key, sizeof(DRBG_ctx.Key));
+		memcpy(DRBG_ctx.V, in_state->V, sizeof(DRBG_ctx.V));
+		DRBG_ctx.reseed_counter = in_state->reseed_counter;
+	}
+}
+
 static void AES256_CTR_DRBG_Update(unsigned char *provided_data, unsigned char *Key, unsigned char *V) {
 	unsigned char temp[48];
 

diff --git a/src/common/rand/rand_nist.h b/src/common/rand/rand_nist.h
@@ -11,6 +11,12 @@
 #include <stddef.h>
 #include <stdint.h>
 
+typedef struct {
+	unsigned char Key[32];
+	unsigned char V[16];
+	int reseed_counter;
+} OQS_NIST_DRBG_struct;
+
 /**
  * Initializes the NIST DRBG with a given seed and with 256-bit security.
  *
@@ -28,4 +34,14 @@ void OQS_randombytes_nist_kat_init_256bit(const uint8_t *entropy_input, const ui
  */
 void OQS_randombytes_nist_kat(uint8_t *random_array, size_t bytes_to_read);
 
+/**
+ * Writes the current state of the NIST DRBG into the provided memory.
+ */
+void OQS_randombytes_nist_kat_get_state(void *out);
+
+/**
+ * Overwrites the current state of the NIST DRBG from the provided memory.
+ */
+void OQS_randombytes_nist_kat_set_state(const void *in);
+
 #endif // OQS_RAND_NIST_H
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -62,7 +62,7 @@ endif()
 add_executable(example_kem example_kem.c)
 target_link_libraries(example_kem PRIVATE ${TEST_DEPS})
 
-add_executable(kat_kem kat_kem.c)
+add_executable(kat_kem kat_kem.c test_helpers.c)
 target_link_libraries(kat_kem PRIVATE ${TEST_DEPS})
 
 add_executable(test_kem test_kem.c)
@@ -80,7 +80,7 @@ set(KEM_TESTS example_kem kat_kem test_kem test_kem_mem speed_kem)
 add_executable(example_sig example_sig.c)
 target_link_libraries(example_sig PRIVATE ${TEST_DEPS})
 
-add_executable(kat_sig kat_sig.c)
+add_executable(kat_sig kat_sig.c test_helpers.c)
 target_link_libraries(kat_sig PRIVATE ${TEST_DEPS})
 
 add_executable(test_sig test_sig.c)
@@ -124,7 +124,8 @@ endif()
 # for DLL builds.
 add_custom_target(
     run_tests
-    COMMAND ${CMAKE_COMMAND} -E env OQS_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON3_EXEC} -m pytest --verbose --numprocesses=auto --ignore=scripts/copy_from_upstream/repos
+    # skip long KAT tests
+    COMMAND ${CMAKE_COMMAND} -E env OQS_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON3_EXEC} -m pytest --verbose --numprocesses=auto --ignore=scripts/copy_from_upstream/repos --ignore=tests/test_kat_all.py
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
     DEPENDS oqs dump_alg_info ${KEM_TESTS} ${SIG_TESTS} ${UNIX_TESTS}
     USES_TERMINAL)