diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 753993e8962c..ed018d0b01f4 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -288,7 +288,9 @@ def load_cmdline_data(fill, cmdline=None): seedfrom = fill.get("seedfrom") if seedfrom: - if seedfrom.startswith(("http://", "https://")): + if seedfrom.startswith( + ("http://", "https://", "ftp://", "ftps://") + ): fill["dsmode"] = sources.DSMODE_NETWORK elif seedfrom.startswith(("file://", "/")): fill["dsmode"] = sources.DSMODE_LOCAL diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c8e1781bd092..6ea8d1541590 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -96,63 +96,80 @@ def get_return_code_from_exception(exc): cause="Invalid url provided", code=NOT_FOUND, headers=None, url=url ) with io.BytesIO() as buffer: + port = url_parts.port or 21 + user = url_parts.username or "anonymous" try: - LOG.debug("Attempting to connect to %s over tls.", url) ftp_tls = ftplib.FTP_TLS( context=create_default_context(), ) + LOG.debug( + "Attempting to connect to %s via port [%s] over tls.", + url, port + ) ftp_tls.connect( host=url_parts.hostname, - port=url_parts.port or 21, + port=port, timeout=timeout or 0.0, # Python docs are wrong about types ) + LOG.debug("Attempting to login with user [%s]", user) ftp_tls.login( - user=url_parts.username or "", + user=user, passwd=url_parts.password or "", ) + LOG.debug("Creating a secure connection", user) ftp_tls.prot_p() + LOG.debug("Reading file: %s", url_parts.path) ftp_tls.retrbinary(f"RETR {url_parts.path}", callback=buffer.write) response = FtpResponse(url_parts.path, contents=buffer) + LOG.debug("Closing connection", url_parts.path) ftp_tls.close() return response except ftplib.all_errors as e: + code = get_return_code_from_exception(e), if "ftps" == url_parts.scheme: raise UrlError( cause=( "Connecting to ftp server over tls " - "failed for url [%s]" + f"failed for url {url} [{code}]" ), - code=get_return_code_from_exception(e), + code=code, headers=None, url=url, ) from e + LOG.info( + "Connecting to ftp server over tls " + "failed for url %s [%s]", url, code + ) try: LOG.debug( "Couldn't connect to %s over tls. Strict mode not " - "required (using protocol 'ftp://' not 'ftps://'), so falling" - "back to ftp", + "required (using protocol 'ftp://' not 'ftps://'), so falling " + "back to ftp. Use 'ftps://' to prevent unencrypted ftp.", url_parts.hostname, ) - LOG.debug("Attempting to connect to %s over tls.", url) ftp = ftplib.FTP() + LOG.debug("Attempting to connect to %s via port %s.", url, port) ftp.connect( host=url_parts.hostname, - port=url_parts.port or 21, + port=port, timeout=timeout or 0.0, # Python docs are wrong about types ) + LOG.debug("Attempting to login with user [%s]", user) ftp.login( - user=url_parts.username or "", + user=user, passwd=url_parts.password or "", ) + LOG.debug("Reading file: %s", url_parts.path) ftp.retrbinary(f"RETR {url_parts.path}", callback=buffer.write) response = FtpResponse(url_parts.path, contents=buffer) + LOG.debug("Closing connection", url_parts.path) ftp.close() return response except ftplib.all_errors as e: raise UrlError( cause=( - "Connecting to ftp server over tls failed for url [%s]" + f"Connecting to ftp server failed for url {url} [{code}]" ), code=get_return_code_from_exception(e), headers=None, diff --git a/integration-requirements.txt b/integration-requirements.txt index 06c768264b7b..dc17759a36b4 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -12,4 +12,3 @@ pytest!=7.3.2 packaging passlib coverage==7.2.7 # Last version supported in Python 3.7 -pyftpdlib diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index 7d1aa5141c06..90c8ac1f68f6 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -104,7 +104,7 @@ def test_nocloud_ftp(client: IntegrationInstance): # hope that users don't see this kind of # test code as an example to follow - client.execute("apt update && apt install python3-pyftpdlib") + client.execute("apt update && apt install -yq python3-pyftpdlib") client.write_to_file( "/server.py", @@ -113,11 +113,11 @@ def test_nocloud_ftp(client: IntegrationInstance): #!/usr/bin/python3 from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler - from pyftpdlib.servers import FTPServe + from pyftpdlib.servers import FTPServer from pyftpdlib.filesystems import UnixFilesystem # yeah, it's not secure but that's not the point - authorizer = DummyAuthorizer + authorizer = DummyAuthorizer() # Define a read-only anonymous user authorizer.add_anonymous("/home/anonymous") @@ -129,7 +129,7 @@ def test_nocloud_ftp(client: IntegrationInstance): server = FTPServer(("localhost", 2121), handler) # start the ftp server - server.run_forever() + server.serve_forever() """ ), ) @@ -141,16 +141,24 @@ def test_nocloud_ftp(client: IntegrationInstance): [Unit] Description=run a local ftp server against Wants=cloud-init-local.service + DefaultDependencies=no + + # we want the network up for network operations + # and NoCloud operates in network timeframe After=systemd-networkd-wait-online.service After=networking.service Before=cloud-init.service [Service] - Type=oneshot + Type=exec ExecStart=/server.py + + [Install] + WantedBy=cloud-init.target """ ) ) + client.execute("chmod 644 /lib/systemd/system/local-ftp.service") client.execute("systemctl enable local-ftp.service") client.execute("mkdir /home/anonymous/") @@ -176,5 +184,5 @@ def test_nocloud_ftp(client: IntegrationInstance): # set the kernel commandline, reboot with it override_kernel_cmdline( - "ci.ds=nocloud;seedfrom=ftp://0.0.0.0:2121", client + "ds=nocloud;seedfrom=ftp://0.0.0.0:2121", client )