Merge pull request #4805 from wasmerio/issue-4804-app-create-templates

CLI: app create - Template handling improvements

lib/backend-api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wasmer-api"
-version = "0.0.28"
+version = "0.0.29"
 description = "Client library for the Wasmer GraphQL API"
 readme = "README.md"
 documentation = "https://docs.rs/wasmer-api"

lib/backend-api/src/query.rs

@@ -13,7 +13,7 @@ use crate::{
     types::{
         self, CreateNamespaceVars, DeployApp, DeployAppConnection, DeployAppVersion,
         DeployAppVersionConnection, DnsDomain, GetAppTemplateFromSlugVariables,
-        GetAppTemplatesQueryVariables, GetCurrentUserWithAppsVars, GetDeployAppAndVersion,
+        GetAppTemplatesVars, GetCurrentUserWithAppsVars, GetDeployAppAndVersion,
         GetDeployAppVersionsVars, GetNamespaceAppsVars, GetSignedUrlForPackageUploadVariables, Log,
         LogStream, PackageVersionConnection, PublishDeployAppVars, PushPackageReleasePayload,
         SignedUrl, TagPackageReleasePayload, UpsertDomainFromZoneFileVars,
@@ -74,18 +74,77 @@ pub async fn fetch_app_templates(
     client: &WasmerClient,
     category_slug: String,
     first: i32,
+    after: Option<String>,
+    sort_by: Option<types::AppTemplatesSortBy>,
 ) -> Result<Option<types::AppTemplateConnection>, anyhow::Error> {
     client
-        .run_graphql_strict(types::GetAppTemplatesQuery::build(
-            GetAppTemplatesQueryVariables {
-                category_slug,
-                first,
-            },
-        ))
+        .run_graphql_strict(types::GetAppTemplates::build(GetAppTemplatesVars {
+            category_slug,
+            first,
+            after,
+            sort_by,
+        }))
         .await
         .map(|r| r.get_app_templates)
 }
 
+/// Fetch all app templates by paginating through the responses.
+///
+/// Will fetch at most `max` templates.
+pub fn fetch_all_app_templates(
+    client: &WasmerClient,
+    page_size: i32,
+    sort_by: Option<types::AppTemplatesSortBy>,
+) -> impl futures::Stream<Item = Result<Vec<types::AppTemplate>, anyhow::Error>> + '_ {
+    let vars = GetAppTemplatesVars {
+        category_slug: String::new(),
+        first: page_size,
+        sort_by,
+        after: None,
+    };
+
+    futures::stream::try_unfold(
+        Some(vars),
+        move |vars: Option<types::GetAppTemplatesVars>| async move {
+            let vars = match vars {
+                Some(vars) => vars,
+                None => return Ok(None),
+            };
+
+            let con = client
+                .run_graphql_strict(types::GetAppTemplates::build(vars.clone()))
+                .await?
+                .get_app_templates
+                .context("backend did not return any data")?;
+
+            let items = con
+                .edges
+                .into_iter()
+                .flatten()
+                .filter_map(|edge| edge.node)
+                .collect::<Vec<_>>();
+
+            let next_cursor = con
+                .page_info
+                .end_cursor
+                .filter(|_| con.page_info.has_next_page);
+
+            let next_vars = next_cursor.map(|after| types::GetAppTemplatesVars {
+                after: Some(after),
+                ..vars
+            });
+
+            #[allow(clippy::type_complexity)]
+            let res: Result<
+                Option<(Vec<types::AppTemplate>, Option<types::GetAppTemplatesVars>)>,
+                anyhow::Error,
+            > = Ok(Some((items, next_vars)));
+
+            res
+        },
+    )
+}
+
 /// Get a signed URL to upload packages.
 pub async fn get_signed_url_for_package_upload(
     client: &WasmerClient,

lib/backend-api/src/types.rs

@@ -9,7 +9,7 @@ mod queries {
 
     use super::schema;
 
-    #[derive(cynic::Scalar, Debug, Clone)]
+    #[derive(cynic::Scalar, Debug, Clone, PartialEq, Eq)]
     pub struct DateTime(pub String);
 
     impl TryFrom<OffsetDateTime> for DateTime {
@@ -149,16 +149,31 @@ mod queries {
         #[arguments(slug: $slug)]
         pub get_app_template: Option<AppTemplate>,
     }
-    #[derive(cynic::QueryVariables, Debug)]
-    pub struct GetAppTemplatesQueryVariables {
+
+    #[derive(cynic::Enum, Clone, Copy, Debug)]
+    pub enum AppTemplatesSortBy {
+        Newest,
+        Oldest,
+        Popular,
+    }
+
+    #[derive(cynic::QueryVariables, Debug, Clone)]
+    pub struct GetAppTemplatesVars {
         pub category_slug: String,
         pub first: i32,
+        pub after: Option<String>,
+        pub sort_by: Option<AppTemplatesSortBy>,
     }
 
     #[derive(cynic::QueryFragment, Debug)]
-    #[cynic(graphql_type = "Query", variables = "GetAppTemplatesQueryVariables")]
-    pub struct GetAppTemplatesQuery {
-        #[arguments(categorySlug: $category_slug, first: $first)]
+    #[cynic(graphql_type = "Query", variables = "GetAppTemplatesVars")]
+    pub struct GetAppTemplates {
+        #[arguments(
+            categorySlug: $category_slug,
+            first: $first,
+            after: $after,
+            sortBy: $sort_by
+        )]
         pub get_app_templates: Option<AppTemplateConnection>,
     }
 
@@ -174,25 +189,32 @@ mod queries {
         pub cursor: String,
     }
 
-    #[derive(cynic::QueryFragment, Debug)]
+    #[derive(serde::Serialize, cynic::QueryFragment, PartialEq, Eq, Debug)]
     pub struct AppTemplate {
+        #[serde(rename = "demoUrl")]
         pub demo_url: String,
         pub language: String,
         pub name: String,
         pub framework: String,
+        #[serde(rename = "createdAt")]
         pub created_at: DateTime,
         pub description: String,
         pub id: cynic::Id,
+        #[serde(rename = "isPublic")]
         pub is_public: bool,
+        #[serde(rename = "repoLicense")]
         pub repo_license: String,
         pub readme: String,
+        #[serde(rename = "repoUrl")]
         pub repo_url: String,
         pub slug: String,
+        #[serde(rename = "updatedAt")]
         pub updated_at: DateTime,
+        #[serde(rename = "useCases")]
         pub use_cases: Jsonstring,
     }
 
-    #[derive(cynic::Scalar, Debug, Clone)]
+    #[derive(cynic::Scalar, Debug, Clone, PartialEq, Eq)]
     #[cynic(graphql_type = "JSONString")]
     pub struct Jsonstring(pub String);
 

lib/cli/Cargo.toml

@@ -145,7 +145,7 @@ virtual-mio = { version = "0.3.1", path = "../virtual-io" }
 # Wasmer-owned dependencies.
 
 webc = { workspace = true }
-wasmer-api = { version = "=0.0.28", path = "../backend-api" }
+wasmer-api = { version = "=0.0.29", path = "../backend-api" }
 edge-schema.workspace = true
 edge-util = { version = "=0.1.0" }
 lazy_static = "1.4.0"

lib/cli/src/commands/app/create.rs

@@ -1,19 +1,28 @@
 //! Create a new Edge app.
 
-use crate::{
-    commands::AsyncCliCommand,
-    opts::{ApiOpts, ItemFormatOpts, WasmerEnv},
-    utils::{load_package_manifest, prompts::PackageCheckMode},
+use std::{
+    collections::HashMap,
+    env,
+    io::Cursor,
+    path::{Path, PathBuf},
+    str::FromStr,
+    time::Duration,
 };
+
 use anyhow::Context;
 use colored::Colorize;
 use dialoguer::{theme::ColorfulTheme, Confirm, Select};
+use futures::stream::TryStreamExt;
 use is_terminal::IsTerminal;
-use std::{collections::HashMap, env, io::Cursor, path::PathBuf, str::FromStr};
 use wasmer_api::{types::AppTemplate, WasmerClient};
 use wasmer_config::{app::AppConfigV1, package::PackageSource};
 
 use super::{deploy::CmdAppDeploy, util::login_user};
+use crate::{
+    commands::AsyncCliCommand,
+    opts::{ApiOpts, ItemFormatOpts, WasmerEnv},
+    utils::{load_package_manifest, prompts::PackageCheckMode},
+};
 
 async fn write_app_config(app_config: &AppConfigV1, dir: Option<PathBuf>) -> anyhow::Result<()> {
     let raw_app_config = app_config.clone().to_yaml()?;
@@ -270,6 +279,106 @@ impl CmdAppCreate {
         Ok(false)
     }
 
+    /// Load cached templates from a file.
+    ///
+    /// Returns an error if the cache file is older than the max age.
+    fn load_cached_templates(
+        path: &Path,
+    ) -> Result<(Vec<AppTemplate>, std::time::Duration), anyhow::Error> {
+        let modified = path.metadata()?.modified()?;
+        let age = modified.elapsed()?;
+
+        let data = std::fs::read_to_string(path)?;
+        match serde_json::from_str::<Vec<AppTemplate>>(&data) {
+            Ok(v) => Ok((v, age)),
+            Err(err) => {
+                std::fs::remove_file(path).ok();
+                Err(err).context("could not deserialize cached file")
+            }
+        }
+    }
+
+    fn persist_template_cache(path: &Path, templates: &[AppTemplate]) -> Result<(), anyhow::Error> {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent).context("could not create cache dir")?;
+        }
+
+        let data = serde_json::to_vec(templates)?;
+
+        std::fs::write(path, data)?;
+        tracing::trace!(path=%path.display(), "persisted app template cache");
+
+        Ok(())
+    }
+
+    /// Tries to retrieve templates from a local file cache.
+    /// Fetches the templates from the backend if the file doesn't exist,
+    /// can't be loaded, or is older then the max age,
+    async fn fetch_templates_cached(
+        client: &WasmerClient,
+        cache_dir: &Path,
+    ) -> Result<Vec<AppTemplate>, anyhow::Error> {
+        const MAX_CACHE_AGE: Duration = Duration::from_secs(60 * 60);
+        const MAX_COUNT: usize = 100;
+        const CACHE_FILENAME: &str = "app_templates.json";
+
+        let cache_path = cache_dir.join(CACHE_FILENAME);
+
+        let cached_items = match Self::load_cached_templates(&cache_path) {
+            Ok((items, age)) => {
+                if age <= MAX_CACHE_AGE {
+                    return Ok(items);
+                }
+                items
+            }
+            Err(e) => {
+                tracing::trace!(error = &*e, "could not load templates from local cache");
+                Vec::new()
+            }
+        };
+
+        // Either no cache present, or cache has exceeded max age.
+        // Fetch the first page.
+        // If first item matches, then no need to re-fetch.
+        let mut stream = Box::pin(wasmer_api::query::fetch_all_app_templates(
+            client,
+            10,
+            Some(wasmer_api::types::AppTemplatesSortBy::Newest),
+        ));
+
+        let first_page = match stream.try_next().await? {
+            Some(items) => items,
+            None => return Ok(Vec::new()),
+        };
+
+        if let (Some(a), Some(b)) = (cached_items.first(), first_page.first()) {
+            if a == b {
+                // Cached items are up to date, no need to query more.
+                return Ok(cached_items);
+            }
+        }
+
+        let mut items = first_page;
+        while let Some(next) = stream.try_next().await? {
+            items.extend(next);
+
+            if items.len() >= MAX_COUNT {
+                break;
+            }
+        }
+
+        // Persist to cache.
+        if let Err(err) = Self::persist_template_cache(&cache_path, &items) {
+            tracing::trace!(error = &*err, "could not persist template cache");
+        }
+
+        // TODO: sort items by popularity!
+        // Since we can't rely on backend sorting because of the cache
+        // preservation logic, the backend needs to add a popluarity field.
+
+        Ok(items)
+    }
+
     // A utility function used to fetch the URL of the template to use.
     async fn get_template_url(&self, client: &WasmerClient) -> anyhow::Result<url::Url> {
         let mut url = if let Some(template) = &self.template {
@@ -287,17 +396,10 @@ impl CmdAppCreate {
                 anyhow::bail!("No template selected")
             }
 
-            let templates: Vec<AppTemplate> =
-                wasmer_api::query::fetch_app_templates(client, String::new(), 10)
-                    .await?
-                    .ok_or(anyhow::anyhow!("No template received from the backend"))?
-                    .edges
-                    .into_iter()
-                    .flatten()
-                    .filter_map(|v| v.node)
-                    .collect();
+            let templates = Self::fetch_templates_cached(client, &self.env.cache_dir).await?;
 
             let theme = ColorfulTheme::default();
+
             let items = templates
                 .iter()
                 .map(|t| {
@@ -315,10 +417,12 @@ impl CmdAppCreate {
                 })
                 .collect::<Vec<_>>();
 
+            // Note: this should really use `dialoger::FuzzySelect`, but that
+            // breaks the formatting.
             let dialog = dialoguer::Select::with_theme(&theme)
                 .with_prompt(format!("Select a template ({} available)", items.len()))
                 .items(&items)
-                .max_length(6)
+                .max_length(10)
                 .clear(true)
                 .report(false)
                 .default(0);