wip tests

cloudinit/sources/DataSourceNoCloud.py

@@ -288,7 +288,9 @@ def load_cmdline_data(fill, cmdline=None):
 
         seedfrom = fill.get("seedfrom")
         if seedfrom:
-            if seedfrom.startswith(("http://", "https://")):
+            if seedfrom.startswith(
+                ("http://", "https://", "ftp://", "ftps://")
+            ):
                 fill["dsmode"] = sources.DSMODE_NETWORK
             elif seedfrom.startswith(("file://", "/")):
                 fill["dsmode"] = sources.DSMODE_LOCAL

cloudinit/url_helper.py

@@ -96,63 +96,80 @@ def get_return_code_from_exception(exc):
             cause="Invalid url provided", code=NOT_FOUND, headers=None, url=url
         )
     with io.BytesIO() as buffer:
+        port = url_parts.port or 21
+        user = url_parts.username or "anonymous"
         try:
-            LOG.debug("Attempting to connect to %s over tls.", url)
             ftp_tls = ftplib.FTP_TLS(
                 context=create_default_context(),
             )
+            LOG.debug(
+                "Attempting to connect to %s via port [%s] over tls.",
+                url, port
+            )
             ftp_tls.connect(
                 host=url_parts.hostname,
-                port=url_parts.port or 21,
+                port=port,
                 timeout=timeout or 0.0,  # Python docs are wrong about types
             )
+            LOG.debug("Attempting to login with user [%s]", user)
             ftp_tls.login(
-                user=url_parts.username or "",
+                user=user,
                 passwd=url_parts.password or "",
             )
+            LOG.debug("Creating a secure connection", user)
             ftp_tls.prot_p()
+            LOG.debug("Reading file: %s", url_parts.path)
             ftp_tls.retrbinary(f"RETR {url_parts.path}", callback=buffer.write)
             response = FtpResponse(url_parts.path, contents=buffer)
+            LOG.debug("Closing connection", url_parts.path)
             ftp_tls.close()
             return response
         except ftplib.all_errors as e:
+            code = get_return_code_from_exception(e),
             if "ftps" == url_parts.scheme:
                 raise UrlError(
                     cause=(
                         "Connecting to ftp server over tls "
-                        "failed for url [%s]"
+                        f"failed for url {url} [{code}]"
                     ),
-                    code=get_return_code_from_exception(e),
+                    code=code,
                     headers=None,
                     url=url,
                 ) from e
+            LOG.info(
+                "Connecting to ftp server over tls "
+                "failed for url %s [%s]", url, code
+            )
         try:
             LOG.debug(
                 "Couldn't connect to %s over tls. Strict mode not "
-                "required (using protocol 'ftp://' not 'ftps://'), so falling"
-                "back to ftp",
+                "required (using protocol 'ftp://' not 'ftps://'), so falling "
+                "back to ftp. Use 'ftps://' to prevent unencrypted ftp.",
                 url_parts.hostname,
             )
-            LOG.debug("Attempting to connect to %s over tls.", url)
 
             ftp = ftplib.FTP()
+            LOG.debug("Attempting to connect to %s via port %s.", url, port)
             ftp.connect(
                 host=url_parts.hostname,
-                port=url_parts.port or 21,
+                port=port,
                 timeout=timeout or 0.0,  # Python docs are wrong about types
             )
+            LOG.debug("Attempting to login with user [%s]", user)
             ftp.login(
-                user=url_parts.username or "",
+                user=user,
                 passwd=url_parts.password or "",
             )
+            LOG.debug("Reading file: %s", url_parts.path)
             ftp.retrbinary(f"RETR {url_parts.path}", callback=buffer.write)
             response = FtpResponse(url_parts.path, contents=buffer)
+            LOG.debug("Closing connection", url_parts.path)
             ftp.close()
             return response
         except ftplib.all_errors as e:
             raise UrlError(
                 cause=(
-                    "Connecting to ftp server over tls failed for url [%s]"
+                    f"Connecting to ftp server failed for url {url} [{code}]"
                 ),
                 code=get_return_code_from_exception(e),
                 headers=None,

integration-requirements.txt

@@ -12,4 +12,3 @@ pytest!=7.3.2
 packaging
 passlib
 coverage==7.2.7  # Last version supported in Python 3.7
-pyftpdlib

tests/integration_tests/datasources/test_nocloud.py

@@ -104,7 +104,7 @@ def test_nocloud_ftp(client: IntegrationInstance):
     # hope that users don't see this kind of
     # test code as an example to follow
 
-    client.execute("apt update && apt install python3-pyftpdlib")
+    client.execute("apt update && apt install -yq python3-pyftpdlib")
 
     client.write_to_file(
         "/server.py",
@@ -113,11 +113,11 @@ def test_nocloud_ftp(client: IntegrationInstance):
             #!/usr/bin/python3
             from pyftpdlib.authorizers import DummyAuthorizer
             from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler
-            from pyftpdlib.servers import FTPServe
+            from pyftpdlib.servers import FTPServer
             from pyftpdlib.filesystems import UnixFilesystem
 
             # yeah, it's not secure but that's not the point
-            authorizer = DummyAuthorizer
+            authorizer = DummyAuthorizer()
 
             # Define a read-only anonymous user
             authorizer.add_anonymous("/home/anonymous")
@@ -129,7 +129,7 @@ def test_nocloud_ftp(client: IntegrationInstance):
             server = FTPServer(("localhost", 2121), handler)
 
             # start the ftp server
-            server.run_forever()
+            server.serve_forever()
             """
         ),
     )
@@ -141,16 +141,24 @@ def test_nocloud_ftp(client: IntegrationInstance):
             [Unit]
             Description=run a local ftp server against
             Wants=cloud-init-local.service
+            DefaultDependencies=no
+
+            # we want the network up for network operations
+            # and NoCloud operates in network timeframe
             After=systemd-networkd-wait-online.service
             After=networking.service
             Before=cloud-init.service
 
             [Service]
-            Type=oneshot
+            Type=exec
             ExecStart=/server.py
+
+            [Install]
+            WantedBy=cloud-init.target
             """
         )
     )
+    client.execute("chmod 644 /lib/systemd/system/local-ftp.service")
     client.execute("systemctl enable local-ftp.service")
 
     client.execute("mkdir /home/anonymous/")
@@ -176,5 +184,5 @@ def test_nocloud_ftp(client: IntegrationInstance):
 
     # set the kernel commandline, reboot with it
     override_kernel_cmdline(
-        "ci.ds=nocloud;seedfrom=ftp://0.0.0.0:2121", client
+        "ds=nocloud;seedfrom=ftp://0.0.0.0:2121", client
     )