early-boot-config: allow gzip compression of user data

README.md

@@ -258,6 +258,9 @@ Here's the user data to change the message of the day setting, as we did in the
 motd = "my own value!"
 ```
 
+If your user data is over the size limit of the platform (e.g. 16KiB for EC2) you can compress the contents with gzip.
+(With [aws-cli](https://aws.amazon.com/cli/), you can use `--user-data fileb:///path/to/gz-file` to pass binary data.)
+
 ### Description of settings
 
 Here we'll describe each setting you can change.

sources/api/early-boot-config/src/provider/aws.rs

@@ -1,6 +1,7 @@
 //! The aws module implements the `PlatformDataProvider` trait for gathering userdata on AWS.
 
 use super::{PlatformDataProvider, SettingsJson};
+use crate::compression::expand_slice_maybe;
 use http::StatusCode;
 use reqwest::blocking::Client;
 use serde_json::json;
@@ -48,7 +49,7 @@ impl AwsDataProvider {
         session_token: &str,
         uri: &str,
         description: &str,
-    ) -> Result<Option<String>> {
+    ) -> Result<Option<Vec<u8>>> {
         debug!("Requesting {} from {}", description, uri);
         let response = client
             .get(uri)
@@ -57,15 +58,35 @@ impl AwsDataProvider {
             .context(error::Request { method: "GET", uri })?;
         trace!("IMDS response: {:?}", &response);
 
+        // IMDS data can be larger than we'd want to log (50k+ compressed) so we don't necessarily
+        // want to show the whole thing, and don't want to show binary data.
+        fn response_string(response: &[u8]) -> String {
+            // arbitrary max len; would be nice to print the start of the data if it's
+            // uncompressed, but we'd need to break slice at a safe point for UTF-8, and without
+            // reading in the whole thing like String::from_utf8.
+            if response.len() > 2048 {
+                "<very long>".to_string()
+            } else if let Ok(s) = String::from_utf8(response.into()) {
+                s
+            } else {
+                "<binary>".to_string()
+            }
+        }
+
         match response.status() {
             code @ StatusCode::OK => {
                 info!("Received {}", description);
-                let response_body = response.text().context(error::ResponseBody {
-                    method: "GET",
-                    uri,
-                    code,
-                })?;
-                trace!("Response text: {:?}", &response_body);
+                let response_body = response
+                    .bytes()
+                    .context(error::ResponseBody {
+                        method: "GET",
+                        uri,
+                        code,
+                    })?
+                    .to_vec();
+
+                let response_str = response_string(&response_body);
+                trace!("Response: {:?}", response_str);
 
                 Ok(Some(response_body))
             }
@@ -74,18 +95,24 @@ impl AwsDataProvider {
             StatusCode::NOT_FOUND => Ok(None),
 
             code @ _ => {
-                let response_body = response.text().context(error::ResponseBody {
-                    method: "GET",
-                    uri,
-                    code,
-                })?;
-                trace!("Response text: {:?}", &response_body);
+                let response_body = response
+                    .bytes()
+                    .context(error::ResponseBody {
+                        method: "GET",
+                        uri,
+                        code,
+                    })?
+                    .to_vec();
+
+                let response_str = response_string(&response_body);
+
+                trace!("Response: {:?}", response_str);
 
                 error::Response {
                     method: "GET",
                     uri,
                     code,
-                    response_body,
+                    response_body: response_str,
                 }
                 .fail()
             }
@@ -98,11 +125,13 @@ impl AwsDataProvider {
         let desc = "user data";
         let uri = Self::USER_DATA_ENDPOINT;
 
-        let user_data_str = match Self::fetch_imds(client, session_token, uri, desc) {
+        let user_data_raw = match Self::fetch_imds(client, session_token, uri, desc) {
             Err(e) => return Err(e),
             Ok(None) => return Ok(None),
             Ok(Some(s)) => s,
         };
+        let user_data_str = expand_slice_maybe(&user_data_raw)
+            .context(error::Decompression { what: "user data" })?;
         trace!("Received user data: {}", user_data_str);
 
         // Remove outer "settings" layer before sending to API
@@ -131,7 +160,9 @@ impl AwsDataProvider {
             match Self::fetch_imds(client, session_token, uri, desc) {
                 Err(e) => return Err(e),
                 Ok(None) => return Ok(None),
-                Ok(Some(s)) => s,
+                Ok(Some(raw)) => {
+                    expand_slice_maybe(&raw).context(error::Decompression { what: "user data" })?
+                }
             }
         };
         trace!("Received instance identity document: {}", iid_str);
@@ -198,6 +229,9 @@ mod error {
         #[snafu(display("Response '{}' from '{}': {}", get_bad_status_code(&source), uri, source))]
         BadResponse { uri: String, source: reqwest::Error },
 
+        #[snafu(display("Failed to decompress {}: {}", what, source))]
+        Decompression { what: String, source: io::Error },
+
         #[snafu(display("Error deserializing from JSON: {}", source))]
         DeserializeJson { source: serde_json::error::Error },
 

sources/api/early-boot-config/src/provider/cdrom.rs

@@ -2,11 +2,13 @@
 //! mounted CDRom.
 
 use super::{PlatformDataProvider, SettingsJson};
+use crate::compression::{expand_file_maybe, expand_slice_maybe, OptionalCompressionReader};
 use serde::Deserialize;
 use snafu::{ensure, OptionExt, ResultExt};
 use std::ffi::OsStr;
-use std::fs::{self, File};
+use std::fs::File;
 use std::io::BufReader;
+use std::iter::FromIterator;
 use std::path::Path;
 
 pub(crate) struct CdromDataProvider;
@@ -52,15 +54,29 @@ impl CdromDataProvider {
             // Since we only look for a specific list of file names, we should never find a file
             // with an extension we don't understand.
             Some(_) => unreachable!(),
-            None => fs::read_to_string(&user_data_file).context(error::InputFileRead {
-                path: user_data_file,
-            })?,
+            None => {
+                // Read the file, decompressing it if compressed.
+                expand_file_maybe(&user_data_file).context(error::InputFileRead {
+                    path: &user_data_file,
+                })?
+            }
         };
 
         if user_data_str.is_empty() {
             return Ok(None);
         }
-        trace!("Received user data: {}", user_data_str);
+
+        // User data could be 700MB compressed!  Eek!  :)
+        if user_data_str.len() <= 2048 {
+            trace!("Received user data: {}", user_data_str);
+        } else {
+            trace!(
+                "Received long user data, starts with: {}",
+                // (this isn't perfect because chars aren't grapheme clusters, but will error
+                // toward printing the whole input, which is fine)
+                String::from_iter(user_data_str.chars().take(2048))
+            );
+        }
 
         // Remove outer "settings" layer before sending to API
         let mut val: toml::Value =
@@ -83,7 +99,7 @@ impl CdromDataProvider {
     fn ovf_user_data<P: AsRef<Path>>(path: P) -> Result<String> {
         let path = path.as_ref();
         let file = File::open(path).context(error::InputFileRead { path })?;
-        let reader = BufReader::new(file);
+        let reader = OptionalCompressionReader::new(BufReader::new(file));
 
         // Deserialize the OVF file, dropping everything we don't care about
         let ovf: Environment =
@@ -109,12 +125,12 @@ impl CdromDataProvider {
             base64_string: base64_str.to_string(),
         })?;
 
-        // Create a valid utf8 str
-        let decoded = std::str::from_utf8(&decoded_bytes).context(error::InvalidUTF8 {
-            base64_string: base64_str.to_string(),
+        // Decompress the data if it's compressed
+        let decoded = expand_slice_maybe(&decoded_bytes).context(error::Decompression {
+            what: "OVF user data",
         })?;
 
-        Ok(decoded.to_string())
+        Ok(decoded)
     }
 }
 
@@ -168,19 +184,12 @@ mod error {
             source: base64::DecodeError,
         },
 
+        #[snafu(display("Failed to decompress {}: {}", what, source))]
+        Decompression { what: String, source: io::Error },
+
         #[snafu(display("Unable to read input file '{}': {}", path.display(), source))]
         InputFileRead { path: PathBuf, source: io::Error },
 
-        #[snafu(display(
-            "Invalid (non-utf8) output from base64 string '{}': {}",
-            base64_string,
-            source
-        ))]
-        InvalidUTF8 {
-            base64_string: String,
-            source: std::str::Utf8Error,
-        },
-
         #[snafu(display("Error serializing TOML to JSON: {}", source))]
         SettingsToJSON { source: serde_json::error::Error },
 

sources/api/early-boot-config/src/provider/local_file.rs

@@ -2,8 +2,8 @@
 //! local file
 
 use super::{PlatformDataProvider, SettingsJson};
+use crate::compression::expand_file_maybe;
 use snafu::{OptionExt, ResultExt};
-use std::fs;
 
 pub(crate) struct LocalFileDataProvider;
 
@@ -16,8 +16,9 @@ impl PlatformDataProvider for LocalFileDataProvider {
         let mut output = Vec::new();
         info!("'{}' exists, using it", Self::USER_DATA_FILE);
 
+        // Read the file, decompressing it if compressed.
         let user_data_str =
-            fs::read_to_string(Self::USER_DATA_FILE).context(error::InputFileRead {
+            expand_file_maybe(Self::USER_DATA_FILE).context(error::InputFileRead {
                 path: Self::USER_DATA_FILE,
             })?;