tests: initial changes to run core suite in uc24 (#13712)

* tests: initial changes to run core suite in uc24

Changes needed to run core test suite in uc24

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from 1c8efb77e1..1db5214d5f

1db5214d5f Improve the remote docs (#36)
2e4a3153a2 1 more comment
3a0fc57e1e add explanation about why we check for ( Do | Doing )
4cf8e635bf fix os.query test after merge
b89b4f8647 fix artifacts name
d30cee6da0 Merge remote-tracking branch 'upstream/main'
5ef5dcbe8f Tests use artifacts in spread tests (#51)
555c43d2ab Support auto-refresh with Do instead of Doing
96c2b0c19c remove tests support for ubuntu 23.04 (EoL)
74082c0c34 Tests improve remote wait (#49)
5121bfb659 remove support for opensuse leap 15.4 (#48)
30df700d08 Add new systems support (#47)
1f08938925 Support check amazon linux version (#46)
43533bdd97 Change the exit value checking for test formats (#45)
3c88244c04 Update check-test-format to support a dir and a list of files (#44)
510d95f429 add extra check for error in auto-refresh detection function
3289d4031b Try open the log with latin-1 encoding when utf-8 is not working
9db785499f improved how the tools are waiting for system reboot
2a5c4414a3 fix shellcheck errors
5e7b63883d Fixes for osquery and tests pkgs (#43)
4c9145e2ac support reboot waiting for auto-refresh
45768f5188 show changes in unknown status after refresh
8013c30c2a Remove support for ubuntu 22.10
b32b80bf54 Fix remote.rait-for test in bionic
5675c625e9 Enable fedora 38
55f4471957 Support for new oss
f2e88b357c New tool used to query spread json reports
cacd35ede0 utils/spread-shellcheck: explain disabled warnings (#42)
c82afb2dee Support --no-install-recommends parameter when installing dependencies with tests.pkgs
b84eea92e2 spread-shellcheck: fix quotes in environment variables (#41)
ab1e51c29f New comparison in os-query for core systems (#40)
e5ae22a5d4 systemd units can be overwritten
63540b845a Fix error messages in remote pull and push
75e8a426a5 make sure the unit is removed in tests.systemd test
9089ff5c02 Update tests to use the new tests.systemd stop-unit
44ecd5e56a Move tests.systemd stop-units to stop-unit
01a2a83b4b Update tests.systemd to have stop units as systemd.sh
162e93bd35 update tests.systemd CLI options to be the same than retry command
14aa43a405 new feature to re-run failed spread tests (#39)
604cb782db Fix shellcheck in systemd tool
bfc71082c8 Update the tests.systemd to allow parameters waiting for service status
8a2d0a99df Adding quiet tool and removing set +-x from tests.pkgs
d90935d2a4 A comment explaining about the default values for wait-for
3232c5dba7 Add support for ubuntu 23.04
a7164fba07 remove fedora 35 support, add fedora 37 support
89b9eb5301 Update systems supported
92bb6a0664 Include snap-sufix in the snaps.name tool

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: 1db5214d5fe91d90b4ffcd4768db8080fcc245ab

* fix core version under test

* adding missing model

* add missing function in nested.sh

* fix keys used for uc24

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from 1db5214d5f..dacfd81de9

dacfd81de9 fix is_core functions

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: dacfd81de95e05a9e56d84be45e0611275b083f4

* use pc-kernel from beta channel

* removing file created for workflow tests

* remove more dirs created during automatic merge

* restore perimssions for files in test snap-repair

* restore tools permissions merged incorrectly

* fix wording in test

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from dacfd81de9..b89ec98b23

b89ec98b23 use local variables in os.query tool

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: b89ec98b239dc9ef729b6af68ce1b5028b4eee23

* fix remove test details

.github/workflows/test.yaml

@@ -465,6 +465,9 @@ jobs:
           - group: ubuntu-core-22
             backend: google
             systems: 'ubuntu-core-22-64'
+          - group: ubuntu-core-24
+            backend: google
+            systems: 'ubuntu-core-24-64'
           - group: ubuntu-arm
             backend: google-arm
             systems: 'ubuntu-20.04-arm-64 ubuntu-core-22-arm-64'
@@ -784,4 +787,4 @@ jobs:
       uses: actions/cache/save@v3
       with:
         path: "${{ github.workspace }}/.test-results"
-        key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.system }}-${{ github.run_attempt }}"
+        key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.system }}-${{ github.run_attempt }}"

spread.yaml

@@ -140,6 +140,10 @@ backends:
                   image: ubuntu-22.04-64
                   workers: 8
                   storage: 20G
+            - ubuntu-core-24-64:
+                  image: ubuntu-24.04-64
+                  workers: 8
+                  storage: 20G
             - ubuntu-secboot-20.04-64:
                   image: ubuntu-20.04-64
                   workers: 1

tests/core/gadget-update-pc/task.yaml

@@ -1,5 +1,10 @@
 summary: Exercise a gadget update on a PC
 
+details: |
+    This test validates the system works properly after the pc gadget
+    snap is installed. Also it is validated the new gadgets are installed
+    correctly.
+
 environment:
     BLOB_DIR: $(pwd)/fake-store-blobdir
     # snap-id of 'pc' gadget snap
@@ -55,7 +60,7 @@ prepare: |
 
     system_seed=""
     system_bios="--system-bios"
-    if os.query is-core20 || os.query is-core22 ; then
+    if os.query is-core-ge 20 ; then
         system_seed="--system-seed"
     fi
     if os.query is-arm; then
@@ -67,7 +72,7 @@ prepare: |
     echo 'this is foo-x2' > foo-x2.img
     cp foo-x2.img pc-snap/foo.img
     echo 'this is foo.cfg' > pc-snap/foo.cfg
-    if os.query is-core20 || os.query is-core22; then
+    if os.query is-core-ge 20; then
         echo 'this is foo-seed.cfg' > pc-snap/foo-seed.cfg
     fi
     sed -i -e 's/^version: \(.*\)-1/version: \1-2/' pc-snap/meta/snap.yaml
@@ -83,7 +88,7 @@ prepare: |
     echo 'this is updated foo-x3' > foo-x3.img
     cp foo-x3.img pc-snap/foo.img
     echo 'this is updated foo.cfg' > pc-snap/foo.cfg
-    if os.query is-core20 || os.query is-core22; then
+    if os.query is-core-ge 20; then
         echo 'this is updated foo-seed.cfg' > pc-snap/foo-seed.cfg
     fi
     echo 'this is bar.cfg' > pc-snap/bar.cfg
@@ -93,7 +98,7 @@ prepare: |
     {"snap-id": "$PC_SNAP_ID", "snap-revision": "$((START_REVISION+2))"}
     EOF
 
-    if os.query is-core20 || os.query is-core22; then
+    if os.query is-core-ge 20; then
         # on uc20, the pc snap we booted with is unasserted for extra debugging
         snap install pc_x1.snap --dangerous
     else
@@ -140,7 +145,7 @@ execute: |
     # - 'EFI System' is mounted at /boot/efi
 
     bootdir=/boot/efi
-    if os.query is-core20 || os.query is-core22; then
+    if os.query is-core-ge 20; then
         # /boot/efi is not mounted on UC20, so use the /run/mnt hierarchy
         bootdir=/run/mnt/ubuntu-boot
     fi
@@ -152,7 +157,7 @@ execute: |
         cp -av pc_x2.snap "$BLOB_DIR/"
 
         
-        if os.query is-core20 || os.query is-core22; then
+        if os.query is-core-ge 20; then
             # on uc20, the pc snap we booted with is unasserted for extra debugging
             snap install pc_x2.snap --dangerous
         else
@@ -195,7 +200,7 @@ execute: |
         snap ack "$p"
         cp -av pc_x3.snap "$BLOB_DIR/"
 
-        if os.query is-core20 || os.query is-core22; then
+        if os.query is-core-ge 20; then
             # on uc20, the pc snap we booted with is unasserted for extra debugging
             snap install pc_x3.snap --dangerous
         else
@@ -227,7 +232,7 @@ execute: |
             test "$(cat foo-updated-written.img)" = 'this is updated foo-x3'
         fi
 
-        if os.query is-core20 || os.query is-core22; then
+        if os.query is-core-ge 20; then
             # a filesystem structure entry was copied to the right place
             test "$(cat /run/mnt/ubuntu-seed/foo-seed.cfg)" = 'this is updated foo-seed.cfg'
 

tests/core/kernel-ver/task.yaml

@@ -1,5 +1,9 @@
 summary: Ensure that we have the right kernel
 
+details: |
+    Check the kernel version is the expected for the current
+    ubuntu core system
+
 # TODO: enable for arm?
 systems: [ubuntu-core-*-64]
 
@@ -13,5 +17,7 @@ execute: |
         VER="^5.4"
     elif os.query is-core22; then
         VER="^5.15"
+    elif os.query is-core24; then
+        VER="^6.8"
     fi
     uname -r | MATCH "$VER"

tests/core/os-release/task.yaml

@@ -1,5 +1,9 @@
 summary: check that os-release is correct
 
+details: |
+    Check the os-release file contains correct information
+    for the current ubuntu core system.
+
 debug: |
     cat /etc/lsb-release || true
 
@@ -13,6 +17,8 @@ execute: |
         MATCH "DISTRIB_RELEASE=20" < /etc/lsb-release
     elif os.query is-core22; then
         MATCH "DISTRIB_RELEASE=22" < /etc/lsb-release
+    elif os.query is-core24; then
+        MATCH "DISTRIB_RELEASE=24" < /etc/lsb-release
     else
         echo "Unknown Ubuntu Core system!"
         exit 1

tests/core/remove/task.yaml

@@ -1,5 +1,10 @@
 summary: Check that removal of essential snaps does not work
 
+details: |
+    This test validates snapd and base snaps cannot be
+    removed. Also checks that a core base snap cannot be 
+    removed when other installed snap is based on that one.
+
 # UC16 does not seed the snapd snap (yet)
 systems: [-ubuntu-core-16-*]
 
@@ -16,6 +21,8 @@ execute: |
         base=core20
     elif os.query is-core22; then
         base=core22
+    elif os.query is-core24; then
+        base=core24
     fi
     echo "Ensure $base cannot be removed"
     if snap remove --purge "$base"; then

tests/core/seed-base-symlinks/task.yaml

@@ -1,11 +1,13 @@
 summary: Check that the seed symlinks work
 
+details: |
+    This test checks that the snap is a symlink into the seed
+
 # We explicitly don't want symlinks from /var/lib/snapd/snaps to
 # /var/lib/snapd/seed in UC20 run mode because /var/lib/snapd/seed snaps are
 # essentially untrusted
 systems:
-  - -ubuntu-core-20-*
-  - -ubuntu-core-22-*
+  - -ubuntu-core-2*
 
 execute: |
     # shellcheck source=tests/lib/systems.sh

tests/core/snap-debug-bootvars/task.yaml

@@ -1,5 +1,9 @@
 summary: Ensure `snap debug bootvars` command works
 
+details: |
+    This test checks the command `snap debug bootvars` displays
+    correct boot variables information.
+
 debug: |
     cat default.out || true
     cat uc20.out || true
@@ -10,7 +14,7 @@ execute: |
     # does not outright fail
     snap debug boot-vars  > default.out
 
-    if os.query is-core20 || os.query is-core22; then
+    if os.query is-core-ge 20; then
         echo "Check the boot-vars default output looks sane"
         # boot-vars default output is for the run mode bootloader, make sure its
         # output looks sane (though we don't expect any of the variables to be

tests/core/snap-repair/task.yaml

@@ -1,5 +1,9 @@
 summary: Ensure that snap-repair works on Ubuntu Core
 
+details: |
+    Checks the `snap repair` command is able repair the
+    system and the repairs are listed properly.
+
 environment:
     BLOB_DIR: $(pwd)/fake-store-blobdir
     STORE_ADDR: localhost:11028
@@ -53,6 +57,9 @@ execute: |
         elif os.query is-core22; then
             REPAIR_SCRIPT=uc22-run.sh
             REPAIR_JSON=uc22-run.json
+        elif os.query is-core24; then
+            REPAIR_SCRIPT=uc24-run.sh
+            REPAIR_JSON=uc24-run.json
         fi
 
         fakestore new-repair --dir "$BLOB_DIR" "$REPAIR_SCRIPT" --repair-json="$PWD/$REPAIR_JSON"
@@ -75,7 +82,7 @@ execute: |
         test "$(snapd.tool exec snap-repair list | tail -n +2 | wc -l)" = 1
 
         echo "And the SNAP_SYSTEM_MODE env var is set appropriately"
-        if os.query is-core20 || os.query is-core22; then
+        if os.query is-core-ge 20; then
             snapd.tool exec snap-repair show testrootorg-1 | MATCH "echo run"
         else
             snapd.tool exec snap-repair show testrootorg-1 | MATCH "echo ''"
@@ -129,6 +136,8 @@ execute: |
             fakestore new-repair --dir "$BLOB_DIR" uc20-recover.sh --repair-json="$PWD/uc20-recover.json"
         elif os.query is-core22; then
             fakestore new-repair --dir "$BLOB_DIR" uc22-recover.sh --repair-json="$PWD/uc22-recover.json"
+        elif os.query is-core24; then
+            fakestore new-repair --dir "$BLOB_DIR" uc24-recover.sh --repair-json="$PWD/uc24-recover.json"
         fi
 
         "$TESTSTOOLS"/store-state setup-fake-store "$BLOB_DIR"

tests/core/snap-repair/uc24-recover.json

@@ -0,0 +1,10 @@
+{
+    "revision": "0",
+    "summary": "recovery mode repair assertion",
+    "bases": [
+        "core24"
+    ],
+    "modes": [
+        "recover"
+    ]
+}

tests/core/snap-repair/uc24-recover.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+
+# show what mode we are in
+echo "$SNAP_SYSTEM_MODE"
+
+# touch a file in /host to show that we did something from recover mode that 
+# persists to run mode
+touch /host/ubuntu-data/system-data/var/lib/snapd/FIXED
+
+# mark us as done using the snap-repair command
+/usr/lib/snapd/snap-repair "done"

tests/core/snap-repair/uc24-run.json

@@ -0,0 +1,9 @@
+{
+    "revision": "0",
+    "bases": [
+        "core24"
+    ],
+    "modes": [
+        "run"
+    ]
+}

tests/core/snap-repair/uc24-run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -ex
+
+# show what mode we are in
+echo "$SNAP_SYSTEM_MODE"
+
+# mark us as done using the env var
+echo "done" >&"$SNAP_REPAIR_STATUS_FD"

tests/lib/assertions/ubuntu-core-24-amd64.model

@@ -0,0 +1,42 @@
+type: model
+authority-id: pxF0fchmU682PEmmay31jjyCHgvscRcj
+series: 16
+brand-id: pxF0fchmU682PEmmay31jjyCHgvscRcj
+model: ubuntu-core-24-amd64-dangerous
+architecture: amd64
+base: core24
+grade: dangerous
+snaps:
+  -
+    default-channel: 24/edge
+    id: UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH
+    name: pc
+    type: gadget
+  -
+    default-channel: 24/edge
+    id: pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza
+    name: pc-kernel
+    type: kernel
+  -
+    default-channel: latest/edge
+    id: dwTAh7MZZ01zyriOZErqd1JynQLiOGvM
+    name: core24
+    type: base
+  -
+    default-channel: latest/edge
+    id: PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4
+    name: snapd
+    type: snapd
+timestamp: 2023-11-09T12:57:29+00:00
+sign-key-sha3-384: Z79zvem9vfFnGDf5ujLe9x4Ztdf1mwOOoxSkY1WWv3cSDh_cyFrRkmFGVaqs8rWc
+
+AcLBcwQAAQoAHRYhBMnGmNEOGnW9TKVit38uBaVTGCUfBQJlTOv0AAoJEH8uBaVTGCUf5AkQAKkW
+O5fUF1F/XPRfE+t8FoIIq3eeOolwCxt5XD2LjJ7oLNYh3YfqXIvitA1lSo3KxT0zIJB9cgUx5ITN
+RMVvRP9jkcFC0K2sUcoEbJAkdXlWpvdWWjaQbmZ+9cVEUQF9QZmF4MMOxPpFs8lDrcc7QfGw+g/7
+xeomXygiC3yAmBrDzfqojJnIPb80u/1qWrSa8DCr09xELK7m30kVwlk9o865nyDQsCh/5RT63/Ja
+7hJEKgihf4VAzOgfEM8tYQ5jFmhvwC3/uaSonoge6/fJBQI+pKKswjxfmQH73xxX9vifkbbX/atj
+e359+YPAzntFuMvoNBA3y3JXaWpOzwZzKsr2csnQzFlcpbwjzrLPAGpkuWbFXlMbX8VyUHZHfMuj
+p/Ro1A7S3OQmdm0dL8L8yzQhrgvA2bPnPUSvgyK4lnzBFwkCzbw93AG9JMtIx0nLiNRgdBeavxaL
+EH42gqVLt22F5oFCfvcv1IMJ92mjJ9Txs4ubFtk9NYoW7hBqxnZoBEZTEAg+DRSF9RAg1c+nhwhi
+hDMuRfjkgvciJYOx6hhYWenHZYqeyHcIDcSWAtKZWS2W81kyIYBo4yT5PaA0+7T727WX6N0zjYFM
+/k8a4NipctTZb9UDIqL9U0zYZmGnhmsixclqm0EwFqgO7l2+6jwcsT8EgFGVJ672DirmyKZs

tests/lib/external/snapd-testing-tools/remote/remote.refresh

@@ -2,8 +2,8 @@
 
 show_help() {
     echo "usage: remote.refresh snap [--channel CHANNEL] <SNAPNAME>"
-    echo "usage: remote.refresh full [--channel CHANNEL]"
-    echo "usage: remote.refresh disable-refreshes"
+    echo "       remote.refresh full [--channel CHANNEL]"
+    echo "       remote.refresh disable-refreshes"
     echo ""
     echo "SNAPNAME: allowed options are: kernel, core, base, gadget, snapd or the snap name"
     echo ""

tests/lib/external/snapd-testing-tools/remote/remote.setup

@@ -4,7 +4,7 @@ CFG_FILE="${REMOTE_CFG_FILE:-$(pwd)/remote.setup.cfg}"
 
 show_help() {
     echo "usage: remote.setup config --host <host> --port <port> --user <USER> [--pass <PASS>] [--cert <CERT>]"
-    echo "usage: remote.setup get-config-path"
+    echo "       remote.setup get-config-path"
     echo ""
     echo "Available options:"
     echo "  -h --help   show this help message."

tests/lib/external/snapd-testing-tools/spread.yaml

@@ -64,7 +64,7 @@ restore: |
 
 suites:
     tests/:
-        summary: Main test suite for snapd-testing-tools
+        summary: Main test suite for snapd-testing-tools project
         prepare: |
             echo "Preparing snapd-testing-tools main suite"
             # Create users for the tests