updata/update_metadata: Adds ability to add waves using TOML files

This commit adds to updata the ability to use TOML-formatted files
to add waves to an existing update. The `--wave-file` flag for
the `add-wave` subcommand allows a user to specify the path to the
file where waves are defined.

sources/parse-datetime/README.md

@@ -9,7 +9,7 @@ This library parses a `DateTime<Utc>` from a string.
 The string can be:
 
 * an `RFC3339` formatted date / time
-* a string with the form `"in <unsigned integer> <unit(s)>"` where
+* a string with the form `"[in] <unsigned integer> <unit(s)>"` where 'in' is optional
    * `<unsigned integer>` may be any unsigned integer and
    * `<unit(s)>` may be either the singular or plural form of the following: `hour | hours`, `day | days`, `week | weeks`
 
@@ -19,6 +19,8 @@ Examples:
 * `"in 2 hours"`
 * `"in 6 days"`
 * `"in 2 weeks"`
+* `"1 hour"`
+* `"7 days"`
 
 ## Colophon
 

sources/parse-datetime/src/lib.rs

@@ -6,7 +6,7 @@ This library parses a `DateTime<Utc>` from a string.
 The string can be:
 
 * an `RFC3339` formatted date / time
-* a string with the form `"in <unsigned integer> <unit(s)>"` where
+* a string with the form `"[in] <unsigned integer> <unit(s)>"` where 'in' is optional
    * `<unsigned integer>` may be any unsigned integer and
    * `<unit(s)>` may be either the singular or plural form of the following: `hour | hours`, `day | days`, `week | weeks`
 
@@ -16,6 +16,8 @@ Examples:
 * `"in 2 hours"`
 * `"in 6 days"`
 * `"in 2 weeks"`
+* `"1 hour"`
+* `"7 days"`
 */
 
 use chrono::{DateTime, Duration, FixedOffset, Utc};
@@ -58,23 +60,25 @@ pub fn parse_datetime(input: &str) -> Result<DateTime<Utc>> {
     // Otherwise, pull apart a request like "in 5 days" to get an exact datetime.
     let mut parts: Vec<&str> = input.split_whitespace().collect();
     ensure!(
-        parts.len() == 3,
+        parts.len() == 3 || parts.len() == 2,
         error::DateArgInvalid {
             input,
-            msg: "expected RFC 3339, or something like 'in 7 days'"
+            msg: "expected RFC 3339, or something like 'in 7 days' or '7 days'"
         }
     );
     let unit_str = parts.pop().unwrap();
     let count_str = parts.pop().unwrap();
-    let prefix_str = parts.pop().unwrap();
 
-    ensure!(
-        prefix_str == "in",
-        error::DateArgInvalid {
-            input,
-            msg: "expected RFC 3339, or prefix 'in', something like 'in 7 days'",
-        }
-    );
+    // the prefix string 'in' is optional
+    if let Some(prefix_str) = parts.pop() {
+        ensure!(
+            prefix_str == "in",
+            error::DateArgInvalid {
+                input,
+                msg: "expected prefix 'in', something like 'in 7 days'",
+            }
+        );
+    }
 
     let count: u32 = count_str.parse().context(error::DateArgCount { input })?;
 
@@ -112,6 +116,9 @@ mod tests {
             "in 0 weeks",
             "in 1 week",
             "in 5000000 weeks",
+            "0 weeks",
+            "1 week",
+            "5000000 weeks",
         ];
 
         for input in inputs {
@@ -121,7 +128,7 @@ mod tests {
 
     #[test]
     fn test_unacceptable_strings() {
-        let inputs = vec!["in", "0 hours", "hours", "in 1 month"];
+        let inputs = vec!["in", "0 hou", "hours", "in 1 month"];
 
         for input in inputs {
             assert!(parse_datetime(input).is_err())

sources/updater/update_metadata/Cargo.toml

@@ -8,6 +8,7 @@ publish = false
 
 [dependencies]
 chrono = { version = "0.4.9", features = ["serde"] }
+parse-datetime = { path = "../../parse-datetime" }
 rand = "0.7.0"
 regex = "1.1"
 semver = { version = "0.9.0", features = ["serde"] }

sources/updater/update_metadata/src/error.rs

@@ -46,6 +46,12 @@ pub enum Error {
     #[snafu(display("Migration {} matches regex but missing name", name))]
     BadRegexName { name: String },
 
+    #[snafu(display("Unable to parse datetime from string '{}': {}", datetime, source))]
+    BadDateTime {
+        datetime: String,
+        source: parse_datetime::Error,
+    },
+
     #[snafu(display("Duplicate key ID: {}", keyid))]
     DuplicateKeyId { backtrace: Backtrace, keyid: u32 },
 
@@ -103,6 +109,12 @@ pub enum Error {
         backtrace: Backtrace,
     },
 
-    #[snafu(display("Waves are not ordered: bound {} occurs before bound {}", next, wave))]
-    WavesUnordered { wave: u32, next: u32 },
+    #[snafu(display("Waves are not ordered; percentages and dates must be in ascending order"))]
+    WavesUnordered,
+
+    #[snafu(display(
+        "`fleet_percentage` must be a value between 1 - 100: value provided: {}",
+        provided
+    ))]
+    InvalidFleetPercentage { provided: u32 },
 }

sources/updater/update_metadata/src/lib.rs

@@ -6,6 +6,7 @@ mod se;
 
 use chrono::{DateTime, Duration, Utc};
 use migrator::MIGRATION_FILENAME_RE;
+use parse_datetime::parse_datetime;
 use rand::{thread_rng, Rng};
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -52,6 +53,19 @@ impl Wave {
     }
 }
 
+/// UpdateWaves is provided for the specific purpose of deserializing
+/// update waves from TOML files
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdateWaves {
+    pub waves: Vec<UpdateWave>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UpdateWave {
+    pub start_after: String,
+    pub fleet_percentage: u32,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Images {
     pub boot: String,
@@ -201,67 +215,99 @@ impl Manifest {
         }
     }
 
+    // Ensures wave dates and bounds are in ascending order.
+    // Update.waves is a BTreeMap which means its keys are always ordered.
+    // If a user has fleet percentages (which have been converted to seeds by
+    // this point) out of order, we will catch it here as the dates will also
+    // be out of order.
     fn validate_updates(updates: &[Update]) -> Result<()> {
         for update in updates {
             let mut waves = update.waves.iter().peekable();
             while let Some(wave) = waves.next() {
                 if let Some(next) = waves.peek() {
-                    ensure!(
-                        wave.1 < next.1,
-                        error::WavesUnordered {
-                            wave: *wave.0,
-                            next: *next.0
-                        }
-                    );
+                    ensure!(wave.1 < next.1, error::WavesUnordered);
                 }
             }
         }
         Ok(())
     }
 
-    /// Adds a wave to update, returns number of matching updates for wave
-    pub fn add_wave(
+    /// Returns Updates matching variant, arch, and version
+    fn get_matching_updates(
         &mut self,
         variant: String,
         arch: String,
         image_version: Version,
-        bound: u32,
-        start: DateTime<Utc>,
-    ) -> Result<usize> {
-        let matching: Vec<&mut Update> = self
-            .updates
+    ) -> Vec<&mut Update> {
+        self.updates
             .iter_mut()
             // Find the update that exactly matches the specified update
             .filter(|update| {
                 update.arch == arch && update.variant == variant && update.version == image_version
             })
-            .collect();
-        let num_matching = matching.len();
-        for update in matching {
-            update.waves.insert(bound, start);
-        }
-        Self::validate_updates(&self.updates)?;
-        Ok(num_matching)
+            .collect()
     }
 
-    pub fn remove_wave(
+    /// Adds a vec of waves to update, returns number of matching updates for wave
+    // Wave format in `manifest.json` is slightly different from the wave structs
+    // provided to this function. For example, if two `UpdateWave` structs are
+    // passed to this function:
+    // [
+    //   UpdateWave { start_after: "1 hour", fleet_percentage: 1 },
+    //   UpdateWave { start_after: "1 day", fleet_percentage: 100},
+    // ]
+    //
+    // The resulting `waves` section of the applicable update looks like:
+    // waves: {
+    //   "0": "<UTC datetime of 1 hour from now>",
+    //   "20": "<UTC datetime of 1 day from now>"
+    // }
+    //
+    // This might look odd until you understand that the first wave begins
+    // at the time specified, and includes seeds 0-19, or 1%, of the seeds
+    // available (`MAX_SEED` in this file). The next wave begins at the time
+    // specified and includes seeds 20-MAX_SEED, or 100% of the rest of the
+    // seeds available. We do this so that the waves input can be more
+    // understandable for human operators, with times relative to when they
+    // start a release, but still have absolute times and seeds that are more
+    // understandable in our update code.
+    pub fn set_waves(
         &mut self,
         variant: String,
         arch: String,
         image_version: Version,
-        bound: u32,
-    ) -> Result<()> {
-        let matching: Vec<&mut Update> = self
-            .updates
-            .iter_mut()
-            .filter(|update| {
-                update.arch == arch && update.variant == variant && update.version == image_version
-            })
-            .collect();
+        waves: &UpdateWaves,
+    ) -> Result<usize> {
+        let matching = self.get_matching_updates(variant, arch, image_version);
+        let num_matching = matching.len();
+
         for update in matching {
-            update.waves.remove(&bound);
+            update.waves.clear();
+
+            // The first wave has a 0 seed
+            let mut seed = 0;
+            for wave in &waves.waves {
+                ensure!(
+                    wave.fleet_percentage > 0 && wave.fleet_percentage <= 100,
+                    error::InvalidFleetPercentage {
+                        provided: wave.fleet_percentage
+                    }
+                );
+
+                let start_time = parse_datetime(&wave.start_after).context(error::BadDateTime {
+                    datetime: &wave.start_after,
+                })?;
+                update.waves.insert(seed, start_time);
+
+                // Get the appropriate seed from the percentage given
+                // First get the percentage as a decimal,
+                let percent = wave.fleet_percentage as f32 / 100 as f32;
+                // then, get seed from the percentage of MAX_SEED as a u32
+                seed = (percent * MAX_SEED as f32) as u32;
+            }
         }
-        Ok(())
+        Self::validate_updates(&self.updates)?;
+        Ok(num_matching)
     }
 }