CI Hardware Tests additions

Makefile

@@ -32,15 +32,18 @@ build-docker:
 run-docker: build-docker
 	$(DOCKER) run --privileged --interactive --rm \
 		--volume /dev:/dev \
-		--volume "$(PWD):/app" \
+		--volume "$(shell pwd):/app" \
 		--env RUST_LOG \
 		--env PYTEST_FLAGS \
 		$(TAG) make run
 
-.PHONY: run-hw
+.PHONY: run-hw run-hw-report
 run-hw: build-docker
 	$(MAKE) run-docker PYTEST_FLAGS="--use-usb-devices $(ALLOWED_UUIDS) -m 'not virtual' $(PYTEST_EXTRA)"
 
+run-hw-report:
+	$(MAKE) run-hw PYTEST_EXTRA="--template=html1/index.html --report report.html --junitxml=report-junit.xml"
+
 $(VENV):
 	python3 -m venv "$(VENV)"
 	"$(VENV)"/bin/pip install --requirement requirements.txt

conftest.py

@@ -36,17 +36,31 @@ def get_version(binary: str) -> str:
             version = "[missing]"
         return version
 
+    def get_up_support(binary: str) -> str:
+        path = os.path.join("bin", binary)
+        up_support = "[error]"
+        if os.path.exists(path):
+            try:
+                output = check_output([path, "--help"])
+                up_support = "yes" if "--user-presence" in output else "no"
+            except:
+                pass
+        return up_support
+
     runner_version = get_version("usbip-runner")
     provisioner_version = get_version("usbip-provisioner")
     if runner_version == provisioner_version:
         version = runner_version
     else:
         version = f"{runner_version}/{provisioner_version}"
     header = f"usbip-runner: {version}"
+    up_prov = get_up_support('usbip-provisioner')
+    up_runner = get_up_support('usbip-runner')
+    header += f", user-presence: {up_runner}" if up_prov == up_runner else f", user-presence: {up_runner}/{up_prov}"
 
     if config.getoption("--upgrade"):
-        runner_version = get_version("usbip-runner")
-        provisioner_version = get_version("usbip-provisioner")
+        runner_version = get_version("usbip-runner-old")
+        provisioner_version = get_version("usbip-provisioner-old")
         if runner_version == provisioner_version:
             version = runner_version
         else:

requirements.txt

@@ -5,6 +5,7 @@ fido2 >=1.1,<2
 pexpect >=4,<5
 pynitrokey ==0.4.38
 pytest >=7,<8
+pytest-reporter-html1
 
 # pyyaml is broken with cython 3
 # see https://github.com/yaml/pyyaml/issues/724

tests/basic.py

@@ -1,12 +1,12 @@
 # Copyright (C) 2022 Nitrokey GmbH
 # SPDX-License-Identifier: CC0-1.0
-
+import logging
 import os
 import pytest
 import random
 import string
-from contextlib import contextmanager
-from pexpect import EOF, spawn
+from contextlib import contextmanager, suppress
+from pexpect import EOF, spawn, ExceptionPexpect, TIMEOUT
 from tempfile import TemporaryDirectory
 from utils.fido2 import Fido2
 from utils.ssh import (
@@ -47,8 +47,8 @@ def verify(self, fido2, credential):
         fido2.authenticate([credential])
 
 
-def test_fido2(touch_device):
-    TestFido2().run(touch_device)
+def test_fido2(device):
+    TestFido2().run(device)
 
 
 class TestFido2Resident(UpgradeTest):
@@ -85,8 +85,8 @@ def verify(self, device, credential):
         p.expect("successfully deleted")
 
 
-def test_fido2_resident(touch_device):
-    TestFido2Resident().run(touch_device)
+def test_fido2_resident(device):
+    TestFido2Resident().run(device)
 
 
 class TestSecrets(UpgradeTest):
@@ -98,8 +98,9 @@ def __init__(self):
 
     def _spawn_with_pin(self, s):
         p = spawn(s)
-        p.expect("Current Password")
-        p.sendline(self.pin)
+        with suppress(EOF, TIMEOUT):
+            p.expect("Current PIN", timeout=5)
+            p.sendline(self.pin)
         return p
 
     def _list_and_get(self, i):
@@ -109,15 +110,18 @@ def _list_and_get(self, i):
         assert "test_totp" in output
 
         p = self._spawn_with_pin("nitropy nk3 secrets get test_hotp")
-        if i == 0:
-            p.expect("755224")
-        else:
-            p.expect("287082")
+        before_buf = p.before.decode("utf-8")
+        logging.getLogger("main").debug(before_buf)
+        otp_code = "755224" if i == 0 else "287082"
+        assert otp_code in before_buf
 
         p = self._spawn_with_pin(
             "nitropy nk3 secrets get test_totp --timestamp 59"
         )
-        p.expect("287082")
+        before_buf = p.before.decode("utf-8")
+        logging.getLogger("main").debug(before_buf)
+        otp_code ="287082"
+        assert otp_code in before_buf
 
     @contextmanager
     def context(self, device):
@@ -152,84 +156,3 @@ def verify(self, device, state):
 def test_secrets(device) -> None:
     TestSecrets().run(device)
 
-
-class TestSsh(UpgradeTest):
-    __test__ = False
-
-    def __init__(self, type: str):
-        self.type = type
-
-    @contextmanager
-    def context(self, device):
-        with TemporaryDirectory() as d:
-            yield (device, d)
-
-    def prepare(self, context):
-        (device, d) = context
-        return keygen(d, self.type)
-
-    def verify(self, context, state):
-        (device, d) = context
-        (key, pubkey) = state
-        (key_path, pubkey_path) = keypair(d, key, pubkey)
-        with authorized_key(pubkey):
-            p = spawn(ssh_command(pubkey_path, "whoami"))
-            p.expect(SSH_USER)
-
-
-@pytest.mark.parametrize("type", SSH_KEY_TYPES)
-def test_ssh(device, type) -> None:
-    TestSsh(type).run(device)
-
-
-class TestSshResident(UpgradeTest):
-    __test__ = False
-
-    def __init__(self, type: str):
-        self.type = type + "-sk"
-        # TODO: PIN generation
-        self.pin = "".join(random.choices(string.digits, k=8))
-
-    @contextmanager
-    def context(self, device):
-        with TemporaryDirectory() as d:
-            yield (device, d)
-
-    def prepare(self, context):
-        (device, d) = context
-        device.set_pin(self.pin)
-        return keygen(d, self.type, resident=True, pin=self.pin)
-
-    def verify(self, context, state):
-        (device, d) = context
-        (key, pubkey) = state
-        (key_path, pubkey_path) = keypair(d, key, pubkey)
-        with authorized_key(pubkey):
-            p = spawn(ssh_command(pubkey_path, "whoami"))
-            p.expect(SSH_USER)
-
-        filename = "id_" + self.type.replace('-', '_') + "_rk"
-        download_dir = os.path.join(d, "download")
-        os.mkdir(download_dir)
-        pwd = os.getcwd()
-        try:
-            os.chdir(download_dir)
-            p = spawn("ssh-keygen -K")
-            p.expect("Enter PIN for authenticator")
-            p.sendline(self.pin)
-            p.expect("Enter passphrase")
-            p.sendline("")
-            p.expect("Enter same passphrase")
-            p.sendline("")
-            p.expect(filename)
-            assert os.path.exists(filename)
-            # TODO: check why the key is partially different
-            # with open(filename, "rb") as f:
-            #     assert f.read() == key
-        finally:
-            os.chdir(pwd)
-
-
-@pytest.mark.parametrize("type", SSH_KEY_TYPES)
-def test_ssh_resident(device, type) -> None:
-    TestSshResident(type).run(device)

tests/basic_ssh.py

@@ -0,0 +1,97 @@
+# Copyright (C) 2022 Nitrokey GmbH
+# SPDX-License-Identifier: CC0-1.0
+import os
+import random
+import string
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+
+import pytest
+from pexpect import spawn
+
+from utils.ssh import (
+    SSH_KEY_TYPES, SSH_USER, authorized_key, keygen, keypair, ssh_command
+)
+from utils.upgrade import UpgradeTest
+
+
+class TestSsh(UpgradeTest):
+    __test__ = False
+
+    def __init__(self, type: str):
+        self.type = type
+
+    @contextmanager
+    def context(self, device):
+        with TemporaryDirectory() as d:
+            yield (device, d)
+
+    def prepare(self, context):
+        (device, d) = context
+        return keygen(d, self.type)
+
+    def verify(self, context, state):
+        (device, d) = context
+        (key, pubkey) = state
+        (key_path, pubkey_path) = keypair(d, key, pubkey)
+        with authorized_key(pubkey):
+            p = spawn(ssh_command(pubkey_path, "whoami"))
+            p.expect(SSH_USER)
+
+
+@pytest.mark.parametrize("type", SSH_KEY_TYPES)
+def test_ssh(device, type) -> None:
+    TestSsh(type).run(device)
+
+
+class TestSshResident(UpgradeTest):
+    __test__ = False
+
+    def __init__(self, type: str):
+        self.type = type + "-sk"
+        # TODO: PIN generation
+        self.pin = "".join(random.choices(string.digits, k=8))
+
+    @contextmanager
+    def context(self, device):
+        with TemporaryDirectory() as d:
+            yield (device, d)
+
+    def prepare(self, context):
+        (device, d) = context
+        device.set_pin(self.pin)
+        return keygen(d, self.type, resident=True, pin=self.pin)
+
+    def verify(self, context, state):
+        (device, d) = context
+        (key, pubkey) = state
+        (key_path, pubkey_path) = keypair(d, key, pubkey)
+        with authorized_key(pubkey):
+            p = spawn(ssh_command(pubkey_path, "whoami"))
+            p.expect(SSH_USER)
+
+        filename = "id_" + self.type.replace('-', '_') + "_rk"
+        download_dir = os.path.join(d, "download")
+        os.mkdir(download_dir)
+        pwd = os.getcwd()
+        try:
+            os.chdir(download_dir)
+            p = spawn("ssh-keygen -K")
+            p.expect("Enter PIN for authenticator")
+            p.sendline(self.pin)
+            p.expect("Enter passphrase")
+            p.sendline("")
+            p.expect("Enter same passphrase")
+            p.sendline("")
+            p.expect(filename)
+            assert os.path.exists(filename)
+            # TODO: check why the key is partially different
+            # with open(filename, "rb") as f:
+            #     assert f.read() == key
+        finally:
+            os.chdir(pwd)
+
+
+@pytest.mark.parametrize("type", SSH_KEY_TYPES)
+def test_ssh_resident(device, type) -> None:
+    TestSshResident(type).run(device)