E2E: add (or expand) ingest/inspect/log/new/reset/search/…

…etc command tests (#883) * E2E, ingest_data_to_player_scores_from_stdio(): add * clippy fixes * E2E, test_ingest_from_stdin(): also check data via the tail command * E2E, ingest_data_to_player_scores_from_stdio(): check data for all ingest steps * E2E, test_ingest_recursive(): add... a part of the test * E2E, test_ingest_with_source_name(): add * E2E, test_inspect_lineage(): add * IngestCommand: add a todo * DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_{1,2,3}: stabilize tests * Rename: LineageCommand -> InspectLineageCommand * E2E: stabilize ingest command tests * KamuCliPuppetExt::list_blocks(): add * E2E, test_inspect_query(): add * E2E, test_inspect_schema(): add * kamu-cli, Login: fix a doc string typo * E2E, test_log(): add * E2E, test_new_{root,derivative}(): add * E2E, test_reset(): add * cli-reference.md: update * test_repo_alias_command -> test_repo_command * E2E, KamuApiServerClientExt::ingest_data(): add * E2E, test_search_multi_user(): add * E2E, test_search_by_name(): add * E2E, test_search_by_repo(): add * E2E, test_sql_command(): add * E2E, test_gc(): add * E2E, test_system_info(): add * E2E, test_system_diagnose(): add * CI: test fixes * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [2] * test_sql_command(): use the default name * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [3] * CHANGELOG.md: update * CI: windows build fixes * E2E, test_tail(): add * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [4] * assert_ingest_data_to_player_scores_from_stdio(): stabilize rows order [5] * pretty_assertions::assert_eq!(): expected first, actual after * Remove extra files * E2E: cover "kamu login" / "kamu logout" commands
kamu-data · Oct 15, 2024 · 40fd121 · 40fd121
1 parent e7f4403
commit 40fd121
Show file tree

Hide file tree

Showing 51 changed files with 2,403 additions and 78 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,19 @@ Recommendation: for ease of reading, use the following order:
   - `kamu init` command
   - `kamu add` command
   - `kamu rename` command
+  - `kamu ingest` command
+  - `kamu inspect` command
+  - `kamu log` command
+  - `kamu new` command
+  - `kamu reset` command
+  - `kamu search` command
+  - `kamu sql` command
+  - `kamu system gc` command
+  - `kamu system info` command
+  - `kamu system diagnose` command
+  - `kamu tail` command
+  - `kamu login` command
+  - `kamu logout` command
 ### Changed
 - `kamu repo alias list`: added JSON output alongside with other formats mentioned in the command's help
 - Private Datasets, `DatasetEntry` integration that will allow us to build dataset indexing

diff --git a/resources/cli-reference.md b/resources/cli-reference.md
@@ -20,7 +20,7 @@ To regenerate this schema from existing code, use the following command:
 * `inspect` — Group of commands for exploring dataset metadata
 * `list [ls]` — List all datasets in the workspace
 * `log` — Shows dataset metadata history
-* `login` — Authentiates with a remote ODF server interactively
+* `login` — Authenticates with a remote ODF server interactively
 * `logout` — Logs out from a remote Kamu server
 * `new` — Creates a new dataset manifest from a template
 * `notebook` — Starts the notebook server for exploring the data in the workspace
@@ -508,7 +508,7 @@ Using a filter to inspect blocks containing query changes of a derivative datase
 
 ## `kamu login`
 
-Authentiates with a remote ODF server interactively
+Authenticates with a remote ODF server interactively
 
 **Usage:** `kamu login [OPTIONS] [SERVER] [COMMAND]`
 

diff --git a/src/adapter/graphql/src/mutations/datasets_mut.rs b/src/adapter/graphql/src/mutations/datasets_mut.rs
@@ -110,6 +110,8 @@ impl DatasetsMut {
     }
 
     // TODO: Multi-tenancy
+    //       https://github.com/kamu-data/kamu-cli/issues/891
+
     // TODO: Multi-tenant resolution for derivative dataset inputs (should it only
     //       work by ID?)
     #[allow(unused_variables)]

diff --git a/src/app/cli/src/cli.rs b/src/app/cli/src/cli.rs
@@ -634,7 +634,7 @@ pub struct Log {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-/// Authentiates with a remote ODF server interactively
+/// Authenticates with a remote ODF server interactively
 #[derive(Debug, clap::Args)]
 pub struct Login {
     #[command(subcommand)]

diff --git a/src/app/cli/src/cli_commands.rs b/src/app/cli/src/cli_commands.rs
@@ -127,7 +127,7 @@ pub fn get_command(
             }
         }
         cli::Command::Inspect(c) => match c.subcommand {
-            cli::InspectSubCommand::Lineage(sc) => Box::new(LineageCommand::new(
+            cli::InspectSubCommand::Lineage(sc) => Box::new(InspectLineageCommand::new(
                 cli_catalog.get_one()?,
                 cli_catalog.get_one()?,
                 cli_catalog.get_one()?,

diff --git a/src/app/cli/src/commands/ingest_command.rs b/src/app/cli/src/commands/ingest_command.rs
@@ -145,6 +145,8 @@ impl Command for IngestCommand {
             _ => Ok(()),
         }?;
 
+        // TODO: `kamu ingest`: implement `--recursive` mode
+        //        https://github.com/kamu-data/kamu-cli/issues/886
         if self.recursive {
             unimplemented!("Sorry, recursive ingest is not yet implemented")
         }

diff --git a/src/app/cli/src/commands/lineage_command.rs → ...i/src/commands/inspect_lineage_command.rs b/src/app/cli/src/commands/lineage_command.rs → ...i/src/commands/inspect_lineage_command.rs
@@ -31,7 +31,7 @@ pub enum LineageOutputFormat {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-pub struct LineageCommand {
+pub struct InspectLineageCommand {
     dataset_repo: Arc<dyn DatasetRepository>,
     provenance_svc: Arc<dyn ProvenanceService>,
     workspace_layout: Arc<WorkspaceLayout>,
@@ -41,7 +41,7 @@ pub struct LineageCommand {
     output_config: Arc<OutputConfig>,
 }
 
-impl LineageCommand {
+impl InspectLineageCommand {
     pub fn new<I>(
         dataset_repo: Arc<dyn DatasetRepository>,
         provenance_svc: Arc<dyn ProvenanceService>,
@@ -94,7 +94,7 @@ impl LineageCommand {
 
 // TODO: Support temporality and evolution
 #[async_trait::async_trait(?Send)]
-impl Command for LineageCommand {
+impl Command for InspectLineageCommand {
     async fn run(&mut self) -> Result<(), CLIError> {
         use futures::{StreamExt, TryStreamExt};
         let mut dataset_handles: Vec<_> = if self.dataset_refs.is_empty() {

diff --git a/src/app/cli/src/commands/log_command.rs b/src/app/cli/src/commands/log_command.rs
@@ -29,6 +29,8 @@ use crate::output::OutputConfig;
 pub enum MetadataLogOutputFormat {
     Shell,
     Yaml,
+    // TODO: `kamu log`: support `--output-format json`
+    //       https://github.com/kamu-data/kamu-cli/issues/887
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/app/cli/src/commands/login_silent_command.rs b/src/app/cli/src/commands/login_silent_command.rs
@@ -16,16 +16,19 @@ use crate::{odf_server, CLIError, Command};
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+#[derive(Debug)]
 pub enum LoginSilentMode {
     OAuth(LoginSilentModeOAuth),
     Password(LoginSilentModePassword),
 }
 
+#[derive(Debug)]
 pub struct LoginSilentModeOAuth {
     pub provider: String,
     pub access_token: String,
 }
 
+#[derive(Debug)]
 pub struct LoginSilentModePassword {
     pub login: String,
     pub password: String,

diff --git a/src/app/cli/src/commands/mod.rs b/src/app/cli/src/commands/mod.rs
@@ -20,9 +20,9 @@ mod delete_command;
 mod gc_command;
 mod ingest_command;
 mod init_command;
+mod inspect_lineage_command;
 mod inspect_query_command;
 mod inspect_schema_command;
-mod lineage_command;
 mod list_command;
 mod log_command;
 mod login_command;
@@ -71,9 +71,9 @@ pub use delete_command::*;
 pub use gc_command::*;
 pub use ingest_command::*;
 pub use init_command::*;
+pub use inspect_lineage_command::*;
 pub use inspect_query_command::*;
 pub use inspect_schema_command::*;
-pub use lineage_command::*;
 pub use list_command::*;
 pub use log_command::*;
 pub use login_command::*;

diff --git a/src/app/cli/src/commands/reset_command.rs b/src/app/cli/src/commands/reset_command.rs
@@ -15,6 +15,8 @@ use opendatafabric::*;
 use super::{CLIError, Command};
 use crate::Interact;
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 pub struct ResetCommand {
     interact: Arc<Interact>,
     dataset_repo: Arc<dyn DatasetRepository>,
@@ -66,3 +68,5 @@ impl Command for ResetCommand {
         Ok(())
     }
 }
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/app/cli/src/commands/tail_command.rs b/src/app/cli/src/commands/tail_command.rs
@@ -17,6 +17,8 @@ use opendatafabric::*;
 use super::{CLIError, Command};
 use crate::output::*;
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 pub struct TailCommand {
     query_svc: Arc<dyn QueryService>,
     dataset_ref: DatasetRef,
@@ -93,3 +95,5 @@ impl Command for TailCommand {
         Ok(())
     }
 }
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/app/cli/src/services/gc_service.rs b/src/app/cli/src/services/gc_service.rs
@@ -9,11 +9,17 @@
 
 use std::sync::Arc;
 
-use chrono::{DateTime, Duration, Utc};
+use chrono::{DateTime, Duration, TimeDelta, Utc};
 use internal_error::{InternalError, ResultIntoInternal};
 
 use crate::WorkspaceLayout;
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+const EVICTION_THRESHOLD: TimeDelta = Duration::hours(24);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 pub struct GcService {
     workspace_layout: Arc<WorkspaceLayout>,
 }
@@ -56,9 +62,6 @@ impl GcService {
     /// Evict stale entries to manage cache size
     #[tracing::instrument(level = "debug", skip_all)]
     pub fn evict_cache(&self) -> Result<GcResult, InternalError> {
-        // TODO: Make const after https://github.com/chronotope/chrono/issues/309
-        // Or make into a config option
-        let eviction_threshold: Duration = Duration::hours(24);
         let now = Utc::now();
         let mut entries_freed = 0;
         let mut bytes_freed = 0;
@@ -69,7 +72,7 @@ impl GcService {
                 let mtime: DateTime<Utc> =
                     chrono::DateTime::from(entry.metadata().int_err()?.modified().int_err()?);
 
-                if (now - mtime) > eviction_threshold {
+                if (now - mtime) > EVICTION_THRESHOLD {
                     if entry.path().is_dir() {
                         bytes_freed += fs_extra::dir::get_size(entry.path()).int_err()?;
                         std::fs::remove_dir_all(entry.path()).int_err()?;
@@ -93,7 +96,11 @@ impl GcService {
     }
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 pub struct GcResult {
     pub entries_freed: usize,
     pub bytes_freed: u64,
 }
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/domain/core/src/services/reset_service.rs b/src/domain/core/src/services/reset_service.rs
@@ -14,6 +14,8 @@ use thiserror::Error;
 use crate::entities::SetRefError;
 use crate::*;
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 #[async_trait::async_trait]
 pub trait ResetService: Send + Sync {
     async fn reset_dataset(
@@ -103,3 +105,5 @@ pub struct OldHeadMismatchError {
     pub current_head: Multihash,
     pub old_head: Multihash,
 }
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/e2e/app/cli/common/src/e2e_harness.rs b/src/e2e/app/cli/common/src/e2e_harness.rs
@@ -9,7 +9,7 @@
 
 use std::future::Future;
 
-use chrono::{DateTime, NaiveTime, Utc};
+use chrono::{DateTime, NaiveTime, TimeZone, Utc};
 use kamu_cli_puppet::extensions::KamuCliPuppetExt;
 use kamu_cli_puppet::{KamuCliPuppet, NewWorkspaceOptions};
 use regex::Regex;
@@ -55,12 +55,18 @@ impl KamuCliApiServerHarnessOptions {
         self
     }
 
-    pub fn with_frozen_system_time(mut self, value: DateTime<Utc>) -> Self {
+    pub fn with_custom_frozen_system_time(mut self, value: DateTime<Utc>) -> Self {
         self.frozen_system_time = Some(value);
 
         self
     }
 
+    pub fn with_frozen_system_time(self) -> Self {
+        let t = Utc.with_ymd_and_hms(2050, 1, 2, 3, 4, 5).unwrap();
+
+        self.with_custom_frozen_system_time(t)
+    }
+
     pub fn with_today_as_frozen_system_time(self) -> Self {
         let today = {
             let now = Utc::now();
@@ -69,7 +75,7 @@ impl KamuCliApiServerHarnessOptions {
                 .unwrap()
         };
 
-        self.with_frozen_system_time(today)
+        self.with_custom_frozen_system_time(today)
     }
 
     pub fn with_kamu_config(mut self, content: &str) -> Self {