feat(apt): skip known /etc/apt/sources.list content

Some image creation software will place known stub files in
/etc/apt/sources.list to inform users in comments of desired
deb822 format.

When deb822 is preferred and /etc/apt/sources.list exists, do
not remove that file if it is has a known safe value.

Add support for Ubuntu's default /etc/apt/sources.list which
has the following content:
 # Ubuntu sources have moved to the /etc/apt/sources.list.d/ubuntu.sources
 # file, which uses the deb822 format. Use deb822-formatted .sources files
 # to manage package sources in the /etc/apt/sources.list.d/ directory.
 # See the sources.list(5) manual page for details.

cloudinit/config/cc_apt_configure.py

@@ -184,6 +184,18 @@
 PRIMARY_ARCHES = ["amd64", "i386"]
 PORTS_ARCHES = ["s390x", "arm64", "armhf", "powerpc", "ppc64el", "riscv64"]
 
+UBUNTU_DEFAULT_APT_SOURCES_LIST = """\
+# Ubuntu sources have moved to the /etc/apt/sources.list.d/ubuntu.sources
+# file, which uses the deb822 format. Use deb822-formatted .sources files
+# to manage package sources in the /etc/apt/sources.list.d/ directory.
+# See the sources.list(5) manual page for details.
+"""
+
+# List of allowed content in /etc/apt/sources.list when features
+# APT_DEB822_SOURCE_LIST_FILE is set. Otherwise issue warning about
+# invalid non-deb822 configuration.
+DEB822_ALLOWED_APT_SOURCES_LIST = {"ubuntu": UBUNTU_DEFAULT_APT_SOURCES_LIST}
+
 
 def get_default_mirrors(
     arch=None,
@@ -681,10 +693,23 @@ def generate_sources_list(cfg, release, mirrors, cloud):
     disabled = disable_suites(cfg.get("disable_suites"), rendered, release)
     util.write_file(aptsrc_file, disabled, mode=0o644)
     if aptsrc_file == apt_sources_deb822 and os.path.exists(apt_sources_list):
-        LOG.warning(
-            "Removing %s to favor deb822 source format", apt_sources_list
+        expected_content = DEB822_ALLOWED_APT_SOURCES_LIST.get(
+            cloud.distro.name
         )
-        util.del_file(apt_sources_list)
+        if expected_content:
+            if expected_content != util.load_text_file(apt_sources_list):
+                LOG.warning(
+                    "Replacing %s to favor deb822 source format",
+                    apt_sources_list,
+                )
+                util.write_file(
+                    apt_sources_list, UBUNTU_DEFAULT_APT_SOURCES_LIST
+                )
+        else:
+            LOG.warning(
+                "Removing %s to favor deb822 source format", apt_sources_list
+            )
+            util.del_file(apt_sources_list)
 
 
 def add_apt_key_raw(key, file_name, hardened=False):

tests/integration_tests/util.py

@@ -71,7 +71,7 @@ def verify_clean_log(log: str, ignore_deprecations: bool = True):
         "thinpool by default on Ubuntu due to LP #1982780",
         "WARNING]: Could not match supplied host pattern, ignoring:",
         # Old Ubuntu cloud-images contain /etc/apt/sources.list
-        "WARNING]: Removing /etc/apt/sources.list to favor deb822 source"
+        "WARNING]: Replacing /etc/apt/sources.list to favor deb822 source"
         " format",
         # https://bugs.launchpad.net/ubuntu/+source/netplan.io/+bug/2041727
         "WARNING]: Running ['netplan', 'apply'] resulted in stderr output: "

tests/unittests/config/test_cc_apt_configure.py

@@ -9,7 +9,7 @@
 import pytest
 
 from cloudinit import features
-from cloudinit.config import cc_apt_configure
+from cloudinit.config import cc_apt_configure as cc_apt
 from cloudinit.config.schema import (
     SchemaValidationError,
     get_schema,
@@ -257,37 +257,68 @@ def test_only_install_needed_packages(
         install_packages = mocker.patch.object(
             mycloud.distro, "install_packages"
         )
-        matcher = re.compile(cc_apt_configure.ADD_APT_REPO_MATCH).search
+        matcher = re.compile(cc_apt.ADD_APT_REPO_MATCH).search
 
         def fake_which(cmd):
             if cmd in already_installed:
                 return "foundit"
             return None
 
-        which = mocker.patch.object(cc_apt_configure.shutil, "which")
+        which = mocker.patch.object(cc_apt.shutil, "which")
         which.side_effect = fake_which
-        cc_apt_configure._ensure_dependencies(cfg, matcher, mycloud)
+        cc_apt._ensure_dependencies(cfg, matcher, mycloud)
         if expected_install:
             install_packages.assert_called_once_with(expected_install)
         else:
             install_packages.assert_not_called()
 
 
 class TestAptConfigure:
+    @pytest.mark.parametrize(
+        "src_content,distro_name,expected_content",
+        (
+            pytest.param(
+                "content",
+                "ubuntu",
+                cc_apt.UBUNTU_DEFAULT_APT_SOURCES_LIST,
+                id="ubuntu_replace_invalid_apt_source_list_with_default",
+            ),
+            pytest.param(
+                "content",
+                "debian",
+                None,
+                id="debian_remove_invalid_apt_source_list",
+            ),
+            pytest.param(
+                cc_apt.UBUNTU_DEFAULT_APT_SOURCES_LIST,
+                "ubuntu",
+                cc_apt.UBUNTU_DEFAULT_APT_SOURCES_LIST,
+                id="ubuntu_no_warning_when_existig_sources_list_content_allowed",
+            ),
+        ),
+    )
     @mock.patch(M_PATH + "get_apt_cfg")
-    def test_remove_source(self, m_get_apt_cfg, caplog, tmpdir):
+    def test_remove_source(
+        self,
+        m_get_apt_cfg,
+        src_content,
+        distro_name,
+        expected_content,
+        caplog,
+        tmpdir,
+    ):
         m_get_apt_cfg.return_value = {
             "sourcelist": f"{tmpdir}/etc/apt/sources.list",
             "sourceparts": f"{tmpdir}/etc/apt/sources.list.d/",
         }
-        cloud = get_cloud("ubuntu")
+        cloud = get_cloud(distro_name)
         features.APT_DEB822_SOURCE_LIST_FILE = True
         sources_file = tmpdir.join("/etc/apt/sources.list")
         deb822_sources_file = tmpdir.join(
-            "/etc/apt/sources.list.d/ubuntu.sources"
+            f"/etc/apt/sources.list.d/{distro_name}.sources"
         )
         Path(sources_file).parent.mkdir(parents=True, exist_ok=True)
-        sources_file.write("content")
+        sources_file.write(src_content)
 
         cfg = {
             "sources_list": """\
@@ -297,6 +328,20 @@ def test_remove_source(self, m_get_apt_cfg, caplog, tmpdir):
 Components: main restricted universe multiverse
 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"""
         }
-        cc_apt_configure.generate_sources_list(cfg, "noble", {}, cloud)
-        assert not sources_file.exists()
+        cc_apt.generate_sources_list(cfg, "noble", {}, cloud)
+        if expected_content is None:
+            assert not sources_file.exists()
+            assert f"Removing {sources_file} to favor deb822" in caplog.text
+        else:
+            if src_content != expected_content:
+                assert (
+                    f"Replacing {sources_file} to favor deb822" in caplog.text
+                )
+
+            assert (
+                cc_apt.UBUNTU_DEFAULT_APT_SOURCES_LIST == sources_file.read()
+            )
+            assert (
+                f"Removing {sources_file} to favor deb822" not in caplog.text
+            )
         assert deb822_sources_file.exists()