feat(schema): use netplan python api for schema validation annotation

cloudinit/cmd/clean.py

@@ -13,6 +13,7 @@
 
 from cloudinit import settings
 from cloudinit.distros import uses_systemd
+from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
 from cloudinit.stages import Init
 from cloudinit.subp import ProcessExecutionError, runparts, subp
 from cloudinit.util import (
@@ -26,7 +27,7 @@
 
 ETC_MACHINE_ID = "/etc/machine-id"
 GEN_NET_CONFIG_FILES = [
-    "/etc/netplan/50-cloud-init.yaml",
+    CLOUDINIT_NETPLAN_FILE,
     "/etc/NetworkManager/conf.d/99-cloud-init.conf",
     "/etc/NetworkManager/conf.d/30-cloud-init-ip6-addr-gen-mode.conf",
     "/etc/NetworkManager/system-connections/cloud-init-*.nmconnection",

cloudinit/config/schema.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import re
+import shutil
 import sys
 import textwrap
 from collections import defaultdict
@@ -34,7 +35,8 @@
 from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with
 from cloudinit.helpers import Paths
 from cloudinit.sources import DataSourceNotFoundException
-from cloudinit.util import error, get_modules_from_dir, load_file
+from cloudinit.temp_utils import mkdtemp
+from cloudinit.util import error, get_modules_from_dir, load_file, write_file
 
 try:
     from jsonschema import ValidationError as _ValidationError
@@ -574,6 +576,110 @@ def validate_cloudconfig_metaschema(validator, schema: dict, throw=True):
         )
 
 
+def network_schema_version(network_config: dict) -> Optional[int]:
+    """Return the version of the network schema when present."""
+    if "network" in network_config:
+        return network_config["network"].get("version")
+    return network_config.get("version")
+
+
+def netplan_validate_network_schema(
+    network_config: dict,
+    strict: bool = False,
+    annotate: bool = False,
+    log_details: bool = True,
+) -> bool:
+    """On systems with netplan, validate network_config schema for file
+
+    Leverage NetplanParser for error annotation line, column and detailed
+    errors.
+
+    @param network_config: Dict of network configuration settings validated
+        against
+    @param strict: Boolean, when True raise SchemaValidationErrors instead of
+       logging warnings.
+    @param annotate: Boolean, when True, print original network_config_file
+        content with error annotations
+    @param log_details: Boolean, when True logs details of validation errors.
+       If there are concerns about logging sensitive userdata, this should
+       be set to False.
+
+    @return: True when schema validation was performed. False when not on a
+        system with netplan and netplan python support.
+    @raises: SchemaValidationError when netplan's parser raises
+        NetplanParserExceptions.
+    """
+    if network_schema_version(network_config) != 2:
+        return False  # Netplan only validates network version 2 config
+
+    try:
+        from netplan import NetplanParserException, Parser  # type: ignore
+    except ImportError:
+        LOG.debug("Skipping netplan schema validation. No netplan available")
+        return False
+
+    # netplan Parser looks at all *.yaml files in the target directory underA
+    # /etc/netplan. cloud-init should only validate schema of the
+    # network-config it generates, so create a <tmp_dir>/etc/netplan
+    # to validate only our network-config.
+    parse_dir = mkdtemp()
+    netplan_file = os.path.join(parse_dir, "etc/netplan/network-config.yaml")
+
+    # Datasource network config can optionally exclude top-level network key
+    net_cfg = deepcopy(network_config)
+    if "network" not in net_cfg:
+        net_cfg = {"network": net_cfg}
+
+    src_content = safeyaml.dumps(net_cfg)
+    write_file(netplan_file, src_content, mode=0o600)
+
+    parser = Parser()
+    errors = []
+    try:
+        # Parse all netplan *.yaml files.load_yaml_heirarchy looks for nested
+        # etc/netplan subdir under "/".
+        parser.load_yaml_hierarchy(parse_dir)
+    except NetplanParserException as e:
+        errors.append(
+            SchemaProblem(
+                "format-l{line}.c{col}".format(line=e.line, col=e.column),
+                f"Invalid netplan schema. {e.message}",
+            )
+        )
+    if os.path.exists(parse_dir):
+        shutil.rmtree(parse_dir)
+    if errors:
+        if strict:
+            if annotate:
+                # Load YAML marks for annotation
+                _, marks = safeyaml.load_with_marks(src_content)
+                print(
+                    annotated_cloudconfig_file(
+                        net_cfg,
+                        src_content,
+                        marks,
+                        schema_errors=errors,
+                    )
+                )
+            raise SchemaValidationError(errors)
+        if log_details:
+            message = _format_schema_problems(
+                errors,
+                prefix=(
+                    f"Invalid {SchemaType.NETWORK_CONFIG.value} provided:\n"
+                ),
+                separator="\n",
+            )
+        else:
+            message = (
+                f"Invalid {SchemaType.NETWORK_CONFIG.value} provided: "
+                "Please run 'sudo cloud-init schema --system' to "
+                "see the schema errors."
+            )
+        LOG.warning(message)
+    return True
+
+
 def validate_cloudconfig_schema(
     config: dict,
     schema: Optional[dict] = None,
@@ -582,7 +688,7 @@ def validate_cloudconfig_schema(
     strict_metaschema: bool = False,
     log_details: bool = True,
     log_deprecations: bool = False,
-):
+) -> bool:
     """Validate provided config meets the schema definition.
 
     @param config: Dict of cloud configuration settings validated against
@@ -608,6 +714,12 @@ def validate_cloudconfig_schema(
     @raises: ValueError on invalid schema_type not in CLOUD_CONFIG or
         NETWORK_CONFIG
     """
+    if schema_type == SchemaType.NETWORK_CONFIG:
+        if netplan_validate_network_schema(
+            network_config=config, strict=strict, log_details=log_details
+        ):
+            # Schema was validated by netplan
+            return True
     if schema is None:
         schema = get_schema(schema_type)
     try:
@@ -618,7 +730,7 @@ def validate_cloudconfig_schema(
             )
     except ImportError:
         LOG.debug("Ignoring schema validation. jsonschema is not present")
-        return
+        return False
 
     validator = cloudinitValidator(schema, format_checker=FormatChecker())
 
@@ -997,21 +1109,25 @@ def validate_cloudconfig_file(
                 f"{schema_type.value} {config_path} is not a YAML dict."
             )
     if schema_type == SchemaType.NETWORK_CONFIG:
-        # Pop optional top-level "network" key when present
-        netcfg = cloudconfig.get("network", cloudconfig)
-        if not netcfg:
+        if not cloudconfig.get("network", cloudconfig):
             print("Skipping network-config schema validation on empty config.")
             return False
-        elif netcfg.get("version") != 1:
+        if netplan_validate_network_schema(
+            network_config=cloudconfig, strict=True, annotate=annotate
+        ):
+            return True  # schema validation performed by netplan
+        if network_schema_version(cloudconfig) != 1:
+            # Validation requires JSON schema definition in
+            # cloudinit/config/schemas/schema-network-config-v1.json
             print(
                 "Skipping network-config schema validation."
                 " No network schema for version:"
-                f" {netcfg.get('version')}"
+                f" {network_schema_version(cloudconfig)}"
             )
             return False
     try:
         if not validate_cloudconfig_schema(
-            cloudconfig, schema, strict=True, log_deprecations=False
+            cloudconfig, schema=schema, strict=True, log_deprecations=False
         ):
             print(
                 f"Skipping {schema_type.value} schema validation."
@@ -1609,12 +1725,12 @@ def get_processed_or_fallback_path(
             schema_type = SchemaType(args.schema_type)
         else:
             schema_type = SchemaType.CLOUD_CONFIG
+        if schema_type == SchemaType.NETWORK_CONFIG:
+            instancedata_type = InstanceDataType.NETWORK_CONFIG
+        else:
+            instancedata_type = InstanceDataType.USERDATA
         config_files.append(
-            InstanceDataPart(
-                InstanceDataType.USERDATA,
-                schema_type,
-                args.config_file,
-            )
+            InstanceDataPart(instancedata_type, schema_type, args.config_file)
         )
     else:
         if os.getuid() != 0:

cloudinit/distros/arch.py

@@ -10,6 +10,7 @@
 from cloudinit import distros, helpers, subp, util
 from cloudinit.distros import PackageList, net_util
 from cloudinit.distros.parsers.hostname import HostnameConf
+from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
 from cloudinit.net.renderer import Renderer
 from cloudinit.net.renderers import RendererNotFoundError
 from cloudinit.settings import PER_INSTANCE
@@ -23,7 +24,7 @@ class Distro(distros.Distro):
     init_cmd = ["systemctl"]  # init scripts
     renderer_configs = {
         "netplan": {
-            "netplan_path": "/etc/netplan/50-cloud-init.yaml",
+            "netplan_path": CLOUDINIT_NETPLAN_FILE,
             "netplan_header": "# generated by cloud-init\n",
             "postcmds": True,
         }

cloudinit/distros/debian.py

@@ -15,6 +15,7 @@
 from cloudinit.distros.package_management.apt import Apt
 from cloudinit.distros.package_management.package_manager import PackageManager
 from cloudinit.distros.parsers.hostname import HostnameConf
+from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
 
 LOG = logging.getLogger(__name__)
 
@@ -34,7 +35,7 @@ class Distro(distros.Distro):
     hostname_conf_fn = "/etc/hostname"
     network_conf_fn = {
         "eni": "/etc/network/interfaces.d/50-cloud-init",
-        "netplan": "/etc/netplan/50-cloud-init.yaml",
+        "netplan": CLOUDINIT_NETPLAN_FILE,
     }
     renderer_configs = {
         "eni": {

cloudinit/distros/mariner.py

@@ -6,6 +6,7 @@
 
 from cloudinit import helpers
 from cloudinit.distros import photon
+from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
 
 LOG = logging.getLogger(__name__)
 
@@ -24,7 +25,7 @@ class Distro(photon.Distro):
     systemd_locale_conf_fn = "/etc/locale.conf"
     resolve_conf_fn = "/etc/systemd/resolved.conf"
 
-    network_conf_fn = {"netplan": "/etc/netplan/50-cloud-init.yaml"}
+    network_conf_fn = {"netplan": CLOUDINIT_NETPLAN_FILE}
     renderer_configs = {
         "networkd": {
             "resolv_conf_fn": resolve_conf_fn,

cloudinit/distros/ubuntu.py

@@ -13,6 +13,7 @@
 
 from cloudinit.distros import PREFERRED_NTP_CLIENTS, debian
 from cloudinit.distros.package_management.snap import Snap
+from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE
 
 
 class Distro(debian.Distro):
@@ -21,7 +22,7 @@ def __init__(self, name, cfg, paths):
         # Ubuntu specific network cfg locations
         self.network_conf_fn = {
             "eni": "/etc/network/interfaces.d/50-cloud-init.cfg",
-            "netplan": "/etc/netplan/50-cloud-init.yaml",
+            "netplan": CLOUDINIT_NETPLAN_FILE,
         }
         self.renderer_configs = {
             "eni": {

cloudinit/net/netplan.py

@@ -18,6 +18,8 @@
 )
 from cloudinit.net.network_state import NET_CONFIG_TO_V2, NetworkState
 
+CLOUDINIT_NETPLAN_FILE = "/etc/netplan/50-cloud-init.yaml"
+
 KNOWN_SNAPD_CONFIG = b"""\
 # This is the initial network config.
 # It can be overwritten by cloud-init or console-conf.
@@ -242,9 +244,7 @@ class Renderer(renderer.Renderer):
     def __init__(self, config=None):
         if not config:
             config = {}
-        self.netplan_path = config.get(
-            "netplan_path", "etc/netplan/50-cloud-init.yaml"
-        )
+        self.netplan_path = config.get("netplan_path", CLOUDINIT_NETPLAN_FILE)
         self.netplan_header = config.get("netplan_header", None)
         self._postcmds = config.get("postcmds", False)
         self.clean_default = config.get("clean_default", True)

cloudinit/stages.py

@@ -1011,15 +1011,14 @@ def should_run_on_boot_event():
         netcfg, src = self._find_networking_config()
         self._write_network_config_json(netcfg)
 
-        if netcfg and netcfg.get("version") == 1:
+        if netcfg:
             validate_cloudconfig_schema(
                 config=netcfg,
                 schema_type=SchemaType.NETWORK_CONFIG,
-                strict=False,
-                log_details=True,
+                strict=False,  # Warnings not raising exceptions
+                log_details=False,  # May have wifi passwords in net cfg
                 log_deprecations=True,
             )
-
         # ensure all physical devices in config are present
         self.distro.networking.wait_for_physdevs(netcfg)