Ubuntu/devel

ChangeLog

@@ -1,3 +1,7 @@
+23.4.2
+ - fix: Handle invalid user configuration gracefully (#4797)
+   (LP: #2051147)
+
 23.4.1
  - fix: Handle systemctl commands when dbus not ready (#4681)
 

cloudinit/cmd/devel/logs.py

@@ -17,11 +17,17 @@
 
 from cloudinit.cmd.devel import read_cfg_paths
 from cloudinit.helpers import Paths
+from cloudinit.stages import Init
 from cloudinit.subp import ProcessExecutionError, subp
 from cloudinit.temp_utils import tempdir
-from cloudinit.util import chdir, copy, ensure_dir, write_file
+from cloudinit.util import (
+    chdir,
+    copy,
+    ensure_dir,
+    get_config_logfiles,
+    write_file,
+)
 
-CLOUDINIT_LOGS = ["/var/log/cloud-init.log", "/var/log/cloud-init-output.log"]
 CLOUDINIT_RUN_DIR = "/run/cloud-init"
 
 
@@ -209,6 +215,8 @@ def collect_logs(tarfile, include_userdata: bool, verbosity=0):
             " Try sudo cloud-init collect-logs\n"
         )
         return 1
+
+    init = Init(ds_deps=[])
     tarfile = os.path.abspath(tarfile)
     log_dir = datetime.utcnow().date().strftime("cloud-init-logs-%Y-%m-%d")
     with tempdir(dir="/tmp") as tmp_dir:
@@ -242,7 +250,8 @@ def collect_logs(tarfile, include_userdata: bool, verbosity=0):
             verbosity=verbosity,
         )
 
-        for log in CLOUDINIT_LOGS:
+        init.read_cfg()
+        for log in get_config_logfiles(init.cfg):
             _collect_file(log, log_dir, verbosity)
         if include_userdata:
             user_data_file = _get_user_data_file()

cloudinit/config/cc_ca_certs.py

@@ -45,6 +45,13 @@
         "ca_cert_config": None,
         "ca_cert_update_cmd": ["update-ca-certificates"],
     },
+    "photon": {
+        "ca_cert_path": "/etc/ssl/certs/",
+        "ca_cert_local_path": "/etc/pki/tls/certs/",
+        "ca_cert_filename": "cloud-init-ca-cert-{cert_index}.crt",
+        "ca_cert_config": None,
+        "ca_cert_update_cmd": ["rehash_ca_certificates.sh"],
+    },
 }
 
 for distro in (
@@ -84,6 +91,7 @@
     "sle-micro",
     "sles",
     "ubuntu",
+    "photon",
 ]
 
 meta: MetaSchema = {
@@ -165,7 +173,7 @@ def disable_default_ca_certs(distro_name, distro_cfg):
     @param distro_name: String providing the distro class name.
     @param distro_cfg: A hash providing _distro_ca_certs_configs function.
     """
-    if distro_name == "rhel":
+    if distro_name in ["rhel", "photon"]:
         remove_default_ca_certs(distro_cfg)
     elif distro_name in ["alpine", "debian", "ubuntu"]:
         disable_system_ca_certs(distro_cfg)

cloudinit/config/schemas/schema-network-config-v1.json

@@ -17,6 +17,7 @@
         },
         "name": {
           "type": "string",
+          "maxLength": 15,
           "description": "Desired device name should be less than 15 characters. Any characters exceeding 15 will be truncated. This is a limitation of the Linux kernel network-device structure."
         },
         "mac_address": {

cloudinit/distros/__init__.py

@@ -148,6 +148,14 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
 
     osfamily: str
     dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd, dhcp.Udhcpc]
+    # Directory where the distro stores their DHCP leases.
+    # The children classes should override this with their dhcp leases
+    # directory
+    dhclient_lease_directory: Optional[str] = None
+    # A regex to match DHCP lease file(s)
+    # The children classes should override this with a regex matching
+    # their lease file name format
+    dhclient_lease_file_regex: Optional[str] = None
 
     def __init__(self, name, cfg, paths):
         self._paths = paths

cloudinit/distros/alpine.py

@@ -33,6 +33,10 @@ class Distro(distros.Distro):
     renderer_configs = {
         "eni": {"eni_path": network_conf_fn, "eni_header": NETWORK_FILE_HEADER}
     }
+    # Alpine stores dhclient leases at following location:
+    # /var/lib/dhcp/dhclient.leases
+    dhclient_lease_directory = "/var/lib/dhcp"
+    dhclient_lease_file_regex = r"dhclient\.leases"
 
     def __init__(self, name, cfg, paths):
         distros.Distro.__init__(self, name, cfg, paths)

cloudinit/distros/amazon.py

@@ -14,5 +14,11 @@
 
 
 class Distro(rhel.Distro):
+    # Amazon Linux 2 stores dhclient leases at following location:
+    # /var/lib/dhclient/dhclient--<iface_name>.leases
+    # Perhaps there could be a UUID in between two "-" in the file name
+    dhclient_lease_directory = "/var/lib/dhcp"
+    dhclient_lease_file_regex = r"dhclient-[\w-]+\.lease"
+
     def update_package_sources(self):
         return None

cloudinit/distros/debian.py

@@ -47,6 +47,10 @@ class Distro(distros.Distro):
             "postcmds": True,
         },
     }
+    # Debian stores dhclient leases at following location:
+    # /var/lib/dhcp/dhclient.<iface_name>.leases
+    dhclient_lease_directory = "/var/lib/dhcp"
+    dhclient_lease_file_regex = r"dhclient\.\w+\.leases"
 
     def __init__(self, name, cfg, paths):
         super().__init__(name, cfg, paths)

cloudinit/distros/freebsd.py

@@ -36,6 +36,10 @@ class Distro(cloudinit.distros.bsd.BSD):
     pkg_cmd_upgrade_prefix = ["pkg", "upgrade"]
     prefer_fqdn = True  # See rc.conf(5) in FreeBSD
     home_dir = "/usr/home"
+    # FreeBSD has the following dhclient lease path:
+    # /var/db/dhclient.leases.<iface_name>
+    dhclient_lease_directory = "/var/db"
+    dhclient_lease_file_regex = r"dhclient.leases.\w+"
 
     @classmethod
     def reload_init(cls, rcs=None):

cloudinit/distros/rhel.py

@@ -29,6 +29,12 @@ class Distro(distros.Distro):
     network_script_tpl = "/etc/sysconfig/network-scripts/ifcfg-%s"
     tz_local_fn = "/etc/localtime"
     usr_lib_exec = "/usr/libexec"
+    # RHEL and derivatives use NetworkManager DHCP client by default.
+    # But if NM is configured with using dhclient ("dhcp=dhclient" statement)
+    # then the following location is used:
+    # /var/lib/NetworkManager/dhclient-<uuid>-<network_interface>.lease
+    dhclient_lease_directory = "/var/lib/NetworkManager"
+    dhclient_lease_file_regex = r"dhclient-[\w-]+\.lease"
     renderer_configs = {
         "sysconfig": {
             "control": "etc/sysconfig/network",

cloudinit/net/dhcp.py

@@ -29,6 +29,9 @@
 LOG = logging.getLogger(__name__)
 
 NETWORKD_LEASES_DIR = "/run/systemd/netif/leases"
+DHCLIENT_FALLBACK_LEASE_DIR = "/var/lib/dhclient"
+# Match something.lease or something.leases
+DHCLIENT_FALLBACK_LEASE_REGEX = r".+\.leases?$"
 UDHCPC_SCRIPT = """#!/bin/sh
 log() {
     echo "udhcpc[$PPID]" "$interface: $2"
@@ -467,51 +470,38 @@ def _trunc_error(cidr, required, remain):
         return static_routes
 
     @staticmethod
-    def get_dhclient_d():
-        # find lease files directory
-        supported_dirs = [
-            "/var/lib/dhclient",
-            "/var/lib/dhcp",
-            "/var/lib/NetworkManager",
-        ]
-        for d in supported_dirs:
-            if os.path.exists(d) and len(os.listdir(d)) > 0:
-                LOG.debug("Using %s lease directory", d)
-                return d
-        return None
-
-    @staticmethod
-    def get_latest_lease(lease_d=None):
-        # find latest lease file
-        if lease_d is None:
-            lease_d = IscDhclient.get_dhclient_d()
-        if not lease_d:
-            return None
-        lease_files = os.listdir(lease_d)
-        latest_mtime = -1
+    def get_latest_lease(lease_dir, lease_file_regex):
         latest_file = None
 
-        # lease files are named inconsistently across distros.
-        # We assume that 'dhclient6' indicates ipv6 and ignore it.
-        # ubuntu:
-        #   dhclient.<iface>.leases, dhclient.leases, dhclient6.leases
-        # centos6:
-        #   dhclient-<iface>.leases, dhclient6.leases
-        # centos7: ('--' is not a typo)
-        #   dhclient--<iface>.lease, dhclient6.leases
-        for fname in lease_files:
-            if fname.startswith("dhclient6"):
-                # avoid files that start with dhclient6 assuming dhcpv6.
+        # Try primary dir/regex, then the fallback ones
+        for directory, regex in (
+            (lease_dir, lease_file_regex),
+            (DHCLIENT_FALLBACK_LEASE_DIR, DHCLIENT_FALLBACK_LEASE_REGEX),
+        ):
+            if not directory:
                 continue
-            if not (fname.endswith((".lease", ".leases"))):
+
+            lease_files = []
+            try:
+                lease_files = os.listdir(directory)
+            except FileNotFoundError:
                 continue
 
-            abs_path = os.path.join(lease_d, fname)
-            mtime = os.path.getmtime(abs_path)
-            if mtime > latest_mtime:
-                latest_mtime = mtime
-                latest_file = abs_path
-        return latest_file
+            latest_mtime = -1
+            for fname in lease_files:
+                if not re.search(regex, fname):
+                    continue
+
+                abs_path = os.path.join(directory, fname)
+                mtime = os.path.getmtime(abs_path)
+                if mtime > latest_mtime:
+                    latest_mtime = mtime
+                    latest_file = abs_path
+
+            # Lease file found, skipping falling back
+            if latest_file:
+                return latest_file
+        return None
 
     @staticmethod
     def parse_dhcp_server_from_lease_file(lease_file) -> Optional[str]:

cloudinit/sources/DataSourceAzure.py

@@ -44,8 +44,8 @@
     get_ip_from_lease_value,
     get_metadata_from_fabric,
     get_system_info,
-    push_log_to_kvp,
     report_diagnostic_event,
+    report_dmesg_to_kvp,
     report_failure_to_fabric,
 )
 from cloudinit.url_helper import UrlError
@@ -313,7 +313,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]:
 
 
 class DataSourceAzure(sources.DataSource):
-
     dsname = "Azure"
     default_update_events = {
         EventScope.NETWORK: {
@@ -1306,6 +1305,7 @@ def _report_failure(
             f"Azure datasource failure occurred: {error.as_encoded_report()}",
             logger_func=LOG.error,
         )
+        report_dmesg_to_kvp()
         reported = kvp.report_failure_to_host(error)
         if host_only:
             return reported
@@ -1365,6 +1365,7 @@ def _report_ready(
 
         :returns: List of SSH keys, if requested.
         """
+        report_dmesg_to_kvp()
         kvp.report_success_to_host()
 
         try:
@@ -1473,7 +1474,7 @@ def activate(self, cfg, is_new_instance):
                 preserve_ntfs=self.ds_cfg.get(DS_CFG_KEY_PRESERVE_NTFS, False),
             )
         finally:
-            push_log_to_kvp(self.sys_cfg["def_log_file"])
+            report_dmesg_to_kvp()
         return
 
     @property
@@ -1822,7 +1823,7 @@ def _redact_password(cnt, fname):
     if not files:
         files = {}
     util.ensure_dir(datadir, dirmode)
-    for (name, content) in files.items():
+    for name, content in files.items():
         fname = os.path.join(datadir, name)
         if "ovf-env.xml" in name:
             content = _redact_password(content, fname)

cloudinit/sources/DataSourceCloudStack.py

@@ -87,10 +87,12 @@ def __init__(self, sys_cfg, distro, paths):
         # Cloudstack has its metadata/userdata URLs located at
         # http://<virtual-router-ip>/latest/
         self.api_ver = "latest"
-        self.vr_addr = get_vr_address()
+
+        self.distro = distro
+        self.vr_addr = get_vr_address(self.distro)
         if not self.vr_addr:
             raise RuntimeError("No virtual router found!")
-        self.metadata_address = "http://%s/" % (self.vr_addr,)
+        self.metadata_address = f"http://{self.vr_addr}/"
         self.cfg = {}
 
     def _get_domainname(self):
@@ -108,7 +110,10 @@ def _get_domainname(self):
             "Falling back to ISC dhclient"
         )
 
-        lease_file = dhcp.IscDhclient.get_latest_lease()
+        lease_file = dhcp.IscDhclient.get_latest_lease(
+            self.distro.dhclient_lease_directory,
+            self.distro.dhclient_lease_file_regex,
+        )
         if not lease_file:
             LOG.debug("Dhclient lease file wasn't found")
             return None
@@ -259,7 +264,7 @@ def get_default_gateway():
     return None
 
 
-def get_vr_address():
+def get_vr_address(distro):
     # Get the address of the virtual router via dhcp leases
     # If no virtual router is detected, fallback on default gateway.
     # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa
@@ -282,7 +287,12 @@ def get_vr_address():
         return latest_address
 
     # Try dhcp lease files next
-    lease_file = dhcp.IscDhclient.get_latest_lease()
+    # get_latest_lease() needs a Distro object to know which directory
+    # stores lease files
+    lease_file = dhcp.IscDhclient.get_latest_lease(
+        distro.dhclient_lease_directory, distro.dhclient_lease_file_regex
+    )
+
     if lease_file:
         latest_address = dhcp.IscDhclient.parse_dhcp_server_from_lease_file(
             lease_file