build: allow validating image-size of resulting images

.github/workflows/build-gluon.yml

@@ -50,6 +50,9 @@ jobs:
       - name: Build Gluon
         run: docker run --rm -v $PWD:/gluon-ci -w /gluon-ci --user "$(id -u):$(id -g)" gluon-ci-container contrib/actions/run-build.sh ${{ matrix.target }}
 
+      - name: Check Image size
+        run: contrib/check-image-size.py --github-actions --overhead 256 output/meta/openwrt-profiles/${{ matrix.target }}.json
+
       - name: Archive build logs
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v3
@@ -63,3 +66,8 @@ jobs:
           name: ${{ matrix.target }}_output
           path: output
 
+      - name: Archive metadata
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.target }}_metadata
+          path: output/meta

Makefile

@@ -43,6 +43,7 @@ GLUON_OUTPUTDIR ?= output
 GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images
 GLUON_PACKAGEDIR ?= $(GLUON_OUTPUTDIR)/packages
 GLUON_DEBUGDIR ?= $(GLUON_OUTPUTDIR)/debug
+GLUON_METADIR ?= $(GLUON_OUTPUTDIR)/meta
 GLUON_TARGETSDIR ?= targets
 GLUON_PATCHESDIR ?= patches
 
@@ -73,7 +74,7 @@ GLUON_VARS = \
 	GLUON_VERSION GLUON_SITE_VERSION \
 	GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_AUTOREMOVE GLUON_DEBUG GLUON_MINIFY GLUON_DEPRECATED \
 	GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR GLUON_DEBUGDIR \
-	GLUON_SITEDIR GLUON_AUTOUPDATER_BRANCH GLUON_AUTOUPDATER_ENABLED GLUON_LANGS GLUON_BASE_FEEDS \
+	GLUON_METADIR GLUON_SITEDIR GLUON_AUTOUPDATER_BRANCH GLUON_AUTOUPDATER_ENABLED GLUON_LANGS GLUON_BASE_FEEDS \
 	GLUON_TARGET BOARD SUBTARGET
 
 unexport $(GLUON_VARS)

contrib/check-image-size.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import sys
+from enum import Enum
+
+# Enum Class for checking image size
+class ImageSizeCheck(Enum):
+    OK = "OK"
+    TOO_BIG = "TOO_BIG"
+    IGNORED = "IGNORED"
+    UNKNOWN = "UNKNOWN"
+
+
+# Some devices pad their images to IMAGE_SIZE and apply a firmware header.
+# Exclude this from the image size check.
+excluded_devices = [
+    "tplink_cpe210-v1",
+    "tplink_cpe210-v2",
+    "tplink_cpe210-v3",
+    "tplink_cpe220-v3",
+    "tplink_cpe510-v1",
+    "tplink_cpe510-v2",
+    "tplink_cpe510-v3",
+    "tplink_cpe710-v1",
+    "tplink_wbs210-v1",
+    "tplink_wbs210-v2",
+    "tplink_wbs510-v1"
+]
+
+
+def open_json(file_path):
+    with open(file_path, 'r') as f:
+        return json.load(f)
+
+
+def load_openwrt_profile_json(json_path):
+    profiles = []
+    profile_json = open_json(json_path)
+    for profile_name, profile_data in profile_json["profiles"].items():
+        device_profile = {
+            "name": profile_name,
+        }
+        if "image_size" in profile_data.get("limits", {}):
+            device_profile["max_image_size"] = profile_data["limits"]["image_size"]
+
+        for image in profile_data["images"]:
+            if image["type"] != "sysupgrade":
+                continue
+            if "size" in image:
+                device_profile["image_size"] = image["size"]
+
+        profiles.append(device_profile)
+
+    return profiles
+
+
+def check_image_size_below_limit(profile, overhead=0):
+    # Skip devices that pad their images
+    if profile["name"] in excluded_devices:
+        return ImageSizeCheck.IGNORED
+
+    if "max_image_size" in profile and "image_size" in profile:
+        if profile["image_size"] + (overhead * 1024) > profile["max_image_size"]:
+            return ImageSizeCheck.TOO_BIG
+        else:	
+            return ImageSizeCheck.OK
+
+    return ImageSizeCheck.UNKNOWN
+
+
+def print_github_actions_warning(message):
+    print('::warning::{}'.format(message))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Check image size of OpenWrt profiles')
+    parser.add_argument(
+        'profile_json',
+        help='Path to profile.json',
+        nargs='+'
+    )
+    parser.add_argument(
+        '--github-actions',
+        help='Generate warnings for use with GitHub Actions',
+        action='store_true'
+    )
+    parser.add_argument(
+        '--overhead',
+        type=int,
+        help='Additional size to add to the image size in kilobyte',
+        default=0
+    )
+    args = parser.parse_args()
+
+    if args.profile_json is None:
+        print('Error: profile.json not specified')
+        sys.exit(1)
+
+    # Load all profile.json files
+    profiles = []
+    for profile_file in args.profile_json:
+        profiles.extend(load_openwrt_profile_json(profile_file))
+
+    # Initialize results with all available ImageSizeCheck values
+    results = {}
+    for check_result in ImageSizeCheck:
+        results[check_result] = []
+
+    for profile in profiles:
+        check_result = check_image_size_below_limit(profile, args.overhead)
+        results[check_result].append(profile)
+
+    for check_result, profiles in results.items():
+        if len(profiles) == 0:
+            continue
+
+        # Group by result type for GitHub Actions
+        if args.github_actions:
+            print('::group::{}'.format(check_result.value))
+
+        for profile in profiles:
+            if check_result == ImageSizeCheck.TOO_BIG:
+                msg = 'Image size of profile {} is too big ({} > {})'.format(
+                    profile["name"],
+                    profile["image_size"] + (args.overhead * 1024),
+                    profile["max_image_size"])
+                print_warning(args.github_actions, msg)
+                if args.github_actions:
+                    print_github_actions_warning(msg)
+                else:
+                    print("Warning: {}".format(msg))
+            elif check_result == ImageSizeCheck.UNKNOWN:
+                msg = 'Image size of profile {} is unknown'.format(
+                    profile["name"])
+                print(msg)
+            elif check_result == ImageSizeCheck.IGNORED:
+                msg = 'Image size of profile {} is ignored (Image size {})'.format(
+                    profile["name"], profile.get("image_size", "unknown"))
+                print(msg)
+            else:
+                msg = 'Image size of profile {} is OK ({} < {})'.format(
+                    profile["name"], 
+                    profile["image_size"] + (args.overhead * 1024),
+                    profile["max_image_size"])
+                print(msg)

patches/openwrt/0008-build-include-size-limits-to-device-metadata.patch

@@ -0,0 +1,107 @@
+From: David Bauer <[email protected]>
+Date: Mon, 11 Dec 2023 14:46:12 +0100
+Subject: build: include size-limits to device-metadata
+
+Include the image and kernel size limitations defined for each device to
+the device metadata JSON.
+
+These informations are only added if defined.
+
+Signed-off-by: David Bauer <[email protected]>
+
+diff --git a/include/image.mk b/include/image.mk
+index dc53fe8b25f268d76e876110d52c06059528c4df..f055f303e835b6fd7e4ba89da57fd60789f16350 100644
+--- a/include/image.mk
++++ b/include/image.mk
+@@ -555,6 +555,8 @@ define Device/Build/initramfs
+ 	VERSION_NUMBER="$(VERSION_NUMBER)" \
+ 	VERSION_CODE="$(VERSION_CODE)" \
+ 	SUPPORTED_DEVICES="$$(SUPPORTED_DEVICES)" \
++	KERNEL_SIZE="$$(KERNEL_SIZE)" \
++	IMAGE_SIZE="$$(IMAGE_SIZE)" \
+ 	$(TOPDIR)/scripts/json_add_image_info.py $$@
+ endef
+ endif
+@@ -686,6 +688,8 @@ define Device/Build/image
+ 	VERSION_NUMBER="$(VERSION_NUMBER)" \
+ 	VERSION_CODE="$(VERSION_CODE)" \
+ 	SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
++	KERNEL_SIZE="$(KERNEL_SIZE)" \
++	IMAGE_SIZE="$(IMAGE_SIZE)" \
+ 	$(TOPDIR)/scripts/json_add_image_info.py $$@
+
+ endef
+@@ -737,6 +741,8 @@ define Device/Build/artifact
+ 	VERSION_NUMBER="$(VERSION_NUMBER)" \
+ 	VERSION_CODE="$(VERSION_CODE)" \
+ 	SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
++	KERNEL_SIZE="$(KERNEL_SIZE)" \
++	IMAGE_SIZE="$(IMAGE_SIZE)" \
+ 	$(TOPDIR)/scripts/json_add_image_info.py $$@
+
+ endef
+diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py
+index aded743bcc744098bfc5ba68fc19dc2885082249..d95aee41f4c0913fdf70cf4898ce9375ad5faa97 100755
+--- a/scripts/json_add_image_info.py
++++ b/scripts/json_add_image_info.py
+@@ -1,6 +1,6 @@
+ #!/usr/bin/env python3
+
+-from os import getenv
++from os import getenv, path
+ from pathlib import Path
+ from sys import argv
+ import hashlib
+@@ -35,6 +35,17 @@ def get_titles():
+     return titles
+
+
++def get_numerical_size(image_size):
++    if image_size.endswith("g"):
++        return int(image_size[:-1]) * 1024 * 1024 * 1024
++    elif image_size.endswith("m"):
++        return int(image_size[:-1]) * 1024 * 1024
++    elif image_size.endswith("k"):
++        return int(image_size[:-1]) * 1024
++    else:
++        return int(image_size)
++
++
+ device_id = getenv("DEVICE_ID")
+
+ sha256_hash = hashlib.sha256()
+@@ -52,6 +63,8 @@ if file_path.with_suffix(file_path.suffix + ".sha256sum").exists():
+ else:
+     hash_unsigned = hash_file
+
++file_size = path.getsize(file_path)
++
+ file_info = {
+     "metadata_version": 1,
+     "target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")),
+@@ -67,6 +80,7 @@ file_info = {
+                     "name": getenv("FILE_NAME"),
+                     "sha256": hash_file,
+                     "sha256_unsigned": hash_unsigned,
++                    "size": file_size,
+                 }
+             ],
+             "device_packages": getenv("DEVICE_PACKAGES").split(),
+@@ -76,6 +90,17 @@ file_info = {
+     },
+ }
+
++if getenv("IMAGE_SIZE") or getenv("KERNEL_SIZE"):
++    file_info["profiles"][device_id]["limits"] = {}
++    if getenv("IMAGE_SIZE"):
++        file_info["profiles"][device_id]["limits"]["image_size"] = get_numerical_size(
++            getenv("IMAGE_SIZE")
++        )
++    if getenv("KERNEL_SIZE"):
++        file_info["profiles"][device_id]["limits"]["kernel_size"] = get_numerical_size(
++            getenv("KERNEL_SIZE")
++        )
++
+ if getenv("FILE_FILESYSTEM"):
+     file_info["profiles"][device_id]["images"][0]["filesystem"] = getenv(
+         "FILE_FILESYSTEM"

scripts/copy_output.lua

@@ -28,6 +28,7 @@ mkdir(env.GLUON_IMAGEDIR..'/factory')
 mkdir(env.GLUON_IMAGEDIR..'/sysupgrade')
 mkdir(env.GLUON_IMAGEDIR..'/other')
 mkdir(env.GLUON_DEBUGDIR)
+mkdir(env.GLUON_METADIR..'/openwrt-profiles')
 
 
 lib.include(target)
@@ -81,6 +82,13 @@ local kernel_debug_dest = string.format('%s/gluon-%s-%s-%s-kernel-debug.tar.zst'
 	target)
 lib.exec {'cp', kernel_debug_source, kernel_debug_dest}
 
+-- copy OpenWrt profile JSON
+local profile_json_source = string.format('openwrt/bin/targets/%s/profiles.json',
+	bindir)
+local profile_json_dest = string.format('%s/openwrt-profiles/%s.json',
+	env.GLUON_METADIR,
+	target)
+lib.exec {'cp', profile_json_source, profile_json_dest}
 
 -- Copy opkg repo
 if (env.GLUON_DEVICES or '') == '' then