From 33af2698a3593e461fe3bb699760c163dd709eab Mon Sep 17 00:00:00 2001 From: Takuya Hashimoto Date: Tue, 17 Nov 2020 19:31:22 +0900 Subject: [PATCH 1/2] Separate dynein configuration from table schema cache --- README.md | 108 +++++++++++-------- src/app.rs | 267 +++++++++++++++++++++++++++++++++-------------- src/bootstrap.rs | 2 +- src/cmd.rs | 24 +++-- src/control.rs | 25 +++-- src/main.rs | 14 +-- 6 files changed, 296 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index ea4dedd3..c8f3c59e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ dynein - DynamoDB CLI - [Quick Start](#quick-start) - [For day-to-day tasks](#for-day-to-day-tasks) - [Installation](#installation) - - [Method 1. HomeBrew (MacOS)](#method-1-homebrew-macos) - - [Method 2. Download a binary](#method-2-download-a-binary) + - [Method 1. Download binaries](#method-1-download-binaries) + - [Method 2. Homebrew (MacOS)](#method-2-homebrew-macos) - [Method 3. Building from source](#method-3-building-from-source) - [How to Use](#how-to-use) - [Prerequisites - AWS Credentials](#prerequisites---aws-credentials) @@ -35,6 +35,8 @@ dynein - DynamoDB CLI - [`dy import`](#dy-import) - [Using DynamoDB Local with `--region local` option](#using-dynamodb-local-with---region-local-option) - [Misc](#misc) + - [Asides](#asides) + - [Troubleshooting](#troubleshooting) - [Ideas for future works](#ideas-for-future-works) @@ -105,7 +107,7 @@ You can move the binary file named "dy" to anywhere under your `$PATH`. ## Prerequisites - AWS Credentials -First of all, please make sure you've already configured AWS Credentials in your environment. dynein depends on [rusoto](https://github.com/rusoto/rusoto) and rusoto [can utilize standard AWS credential toolchains](https://github.com/rusoto/rusoto/blob/master/AWS-CREDENTIALS.md) - for example `~/.aws/credentials` file, [IAM EC2 Instance Profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html), or environment variables such as `AWS_DEFAULT_REGION / AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY`. +First of all, please make sure you've already configured AWS Credentials in your environment. dynein depends on [rusoto](https://github.com/rusoto/rusoto) and rusoto [can utilize standard AWS credential toolchains](https://github.com/rusoto/rusoto/blob/master/AWS-CREDENTIALS.md) - for example `~/.aws/credentials` file, [IAM EC2 Instance Profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html), or environment variables such as `AWS_DEFAULT_REGION / AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_PROFILE`. One convenient way to check if your AWS credential configuration is ok to use dynein is to install and try to execute [AWS CLI](https://aws.amazon.com/cli/) in your environment (e.g. `$ aws dynamodb list-tables`). Once you've [configured AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html), you should be ready to use dynein. @@ -145,7 +147,7 @@ Here `Name` is [a primary key](https://docs.aws.amazon.com/amazondynamodb/latest You don't want to pass `--region` and `--table` everytime? Let's mark the table as "currently using" with the command `dy use`. ``` -$ dy use --region us-west-2 --table Forum +$ dy use Forum --region us-west-2 ``` Now you can interact with the table without specifying a target. @@ -163,32 +165,28 @@ To find more features, `dy help` will show you complete list of available comman $ dy --help dynein x.x.x dynein is a command line tool to interact with DynamoDB tables/data using concise interface. +dynein looks for config files under $HOME/.dynein/ directory. USAGE: dy [OPTIONS] FLAGS: - -h, --help - Prints help information - - -V, --version - Prints version information - + -h, --help Prints help information + -V, --version Prints version information OPTIONS: - -r, --region - The region to use. When using DynamodB Local, `--region local` You can use --region option in both top-level - and subcommand-level - -t, --table - Target table. By executing `$ dy use -t
` you can omit --table on every command. You can use --table - option in both top-level and subcommand-level + -r, --region The region to use (e.g. --region us-east-1). When using DynamodB Local, use `--region + local`. You can use --region option in both top-level and subcommand-level + -t, --table
Target table of the operation. You can use --table option in both top-level and subcommand- + level. You can store table schema locally by executing `$ dy use`, after that + you need not to specify --table on every command SUBCOMMANDS: - admin Admin operations such as creating/updating table or index + admin Admin operations such as creating/updating table or GSI backup Take backup of a DynamoDB table using on-demand backup bootstrap Create sample tables and load test data for bootstrapping bwrite Put or Delete multiple items at one time, up to 25 requests. [API: BatchWriteItem] - config Manage configuration file (~/.dynein/config.yml) from command line + config Manage configuration files (config.yml and cache.yml) from command line del Delete an existing item. [API: DeleteItem] desc Show detailed information of a table. [API: DescribeTable] export Export items from a DynamoDB table and save them as CSV/JSON file @@ -201,7 +199,8 @@ SUBCOMMANDS: restore Restore a DynamoDB table from backup data scan Retrieve items in a table without any condition. [API: Scan] upd Update an existing item. [API: UpdateItem] - use Switch target table context. You can overwrite the context with --table + use Switch target table context. After you use the command you don't need to specify table every time, + but you may overwrite the target table with --table (-t) option ``` dynein consists of multiple layers of subcommands. For example, `dy admin` and `dy config` require you to give additional action to run. @@ -214,15 +213,9 @@ dy-admin x.x.x USAGE: dy admin [OPTIONS] -FLAGS: - -h, --help Prints help information - -V, --version Prints version information +FLAGS: ... -OPTIONS: - -r, --region The region to use. When using DynamodB Local, `--region local` You can use --region option - in both top-level and subcommand-level - -t, --table
Target table. By executing `$ dy use -t
` you can omit --table on every command. You - can use --table option in both top-level and subcommand-level +OPTIONS: ... SUBCOMMANDS: create Create new DynamoDB table or GSI @@ -272,7 +265,7 @@ Now all tables have sample data. Try following commands to play with dynein. Enj $ dy --region us-west-2 use --table Thread $ dy scan -After you 'use' a table like above, dynein assume you're using the same region & table, which info is stored at ~/.dynein/config.yml +After you 'use' a table like above, dynein assume you're using the same region & table, which info is stored at ~/.dynein/config.yml and ~/.dynein/cache.yml Let's move on with the 'us-west-2' region you've just 'use'd... $ dy scan --table Forum $ dy scan -t ProductCatalog @@ -310,7 +303,7 @@ created_at: "2020-03-03T13:34:43+00:00" After the table get ready (= `ACTIVE` status), you can write-to and read-from the table. ``` -$ dy use -t app_users +$ dy use app_users $ dy desc --- name: app_users @@ -389,24 +382,31 @@ However, dynein assume that tipically you're interested in only one table at som By using `dy use` for a table, you can call commands such as `scan`, `get`, `query`, and `put` without specifying table name. ``` -$ dy use -t customers +$ dy use customers $ dy scan ... display items in the "customers" table ... ``` -In detail, when you execute `dy use` command, dynein saves your table usage information under `~/.dynein/config.yml`. You can dump the info with `dy config dump` command. +In detail, when you execute `dy use` command, dynein saves your table usage information in `~/.dynein/config.yml` and caches table schema in `~/.dynein/cache.yml`. You can dump them with `dy config dump` command. ``` +$ ls ~/.dynein/ +cache.yml config.yml + $ dy config dump --- -table: - region: ap-northeast-1 - name: customers - pk: - name: user_id - kind: S - sk: ~ - indexes: ~ +tables: + ap-northeast-1/customers: + region: ap-northeast-1 + name: customers + pk: + name: user_id + kind: S + sk: ~ + indexes: ~ +--- +using_region: ap-northeast-1 +using_table: customers ``` To clear current table configuration, simply execute `dy config clear`. @@ -415,7 +415,10 @@ To clear current table configuration, simply execute `dy config clear`. $ dy config clear $ dy config dump --- -table: ~ +tables: ~ +--- +using_region: ~ +using_table: ~ ``` @@ -426,10 +429,10 @@ As an example let's assume you have [official "Movie" sample data](https://docs. ``` $ dy bootstrap --sample movie ... wait some time while dynein loading data ... -$ dy use -t Movie +$ dy use Movie ``` -After executing `dy use -t ` command, dynein recognize keyscheme and data type of the table. It means that some of the arguments you need to pass to access data (items) is automatically inferred when possible. +After executing `dy use ` command, dynein recognize keyscheme and data type of the table. It means that some of the arguments you need to pass to access data (items) is automatically inferred when possible. Before diving deep into each command, let me describe DynamoDB's "reserved words". One of the traps that beginners can easily fall into is that you cannot use [certain reserved words](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ReservedWords.html) in DynamoDB APIs. DynamoDB reserved words contains common words that you may want to use in your application. For example "name", "year", "url", "token", "error", "date", "group" -- all of them are reserved so you cannot use them in expressions directly. @@ -547,7 +550,7 @@ dynein provides subcommands to write to DynamoDB tables as well. ``` $ dy admin create table write_test --keys id,N -$ dy use -t write_test +$ dy use write_test $ dy put 123 Successfully put an item to the table 'write_test'. @@ -704,7 +707,7 @@ count: 0 size_bytes: 0 created_at: "2020-06-02T14:22:56+00:00" -$ dy use -t app_users +$ dy use app_users $ dy scan --index top_rank_users_index ``` @@ -790,8 +793,25 @@ $ dy scan # Misc +## Asides + dynein is named after [a motor protein](https://en.wikipedia.org/wiki/Dynein). +## Troubleshooting + +If you encounter troubles, the first option worth trying is removing files in `~/.dynein/` or the directory itself. Doing this just clears "cached" info stored locally for dynein and won't affect your data stored in DynamoDB tables. + +``` +$ rm -rf ~/.dynein/ +``` + +To see verbose output for troubleshooting purpose, you can change log level by `RUST_LOG` environment variable. For example: + +``` +$ RUST_LOG=debug RUST_BACKTRACE=1 dy scan --table your_table +``` + + ## Ideas for future works - `dy admin update table` command diff --git a/src/app.rs b/src/app.rs index e3c21da6..f7521839 100644 --- a/src/app.rs +++ b/src/app.rs @@ -20,12 +20,15 @@ use log::{debug,error,info}; use rusoto_core::Region; use rusoto_dynamodb::*; use serde_yaml::Error as SerdeYAMLError; -use std::error; -use std::fmt::{self, Display, Formatter, Error as FmtError}; -use std::fs; -use std::io::Error as IOError; -use std::path; -use std::str::FromStr; +use std::{ + collections::HashMap, + error, + fmt::{self, Display, Formatter, Error as FmtError}, + fs, + io::Error as IOError, + path, + str::FromStr, +}; use super::control; @@ -35,6 +38,13 @@ use super::control; const CONFIG_DIR: &'static str = ".dynein"; const CONFIG_FILE_NAME: &'static str = "config.yml"; +const CACHE_FILE_NAME: &'static str = "cache.yml"; + + +pub enum DyneinFileType { + ConfigFile, + CacheFile, +} #[derive(Serialize, Deserialize, Debug, Clone)] @@ -131,25 +141,35 @@ impl fmt::Display for Messages { write!(f, "{}", match self { Messages::NoEffectiveTable => " To execute commands you must specify target table one of following ways: - * $ dy --region use ... save target region and table. After that you don't need to pass region/table [RECOMMENDED]. - * $ dy --region --table scan ... data operations like 'scan', use --region and --table options. - * $ dy --region desc ... to describe a table, you have to specify table name. -To list all tables in all regions, try: + * [RECOMMENDED] $ dy use ... save target region and table. + * Or, optionally you can pass --region (-r) and --table (-t) options every time to specify target for your commands. +To find all tables in all regions, try: * $ dy ls --all-regions", }) } } -/* -* This is a struct for dynein configuration. -* Currently the only information in the config file is the table information that being used, -* but in future it's possible that we separate cached table information from configurations, -* which may contain "expiration time for cached table", "default region", and "default table name" etc. -*/ -#[derive(Serialize, Deserialize, Debug, Clone)] +/// Config is saved at `~/.dynein/config.yml`. +/// using_region and using_table are changed when you execute `dy use` command. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Config { - pub table: Option, + pub using_region: Option, + pub using_table: Option, + // pub cache_expiration_time: Option, // in second. default 300 (= 5 minutes) +} + +/// Cache is saved at `~/.dynein/cache.yml` +/// Cache contains retrieved info of tables, and how fresh they are (cache_created_at). +/// Currently Cache struct doesn't manage freshness of each table. +/// i.e. Entire cache will be removed after cache_expiration_time in Config has passed. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct Cache { + /// cached table schema information. + /// table schemas are stored in keys to identify the target table "/" -- e.g. "ap-northeast-1/Employee" + pub tables: Option>, + // pub cache_updated_at: String, + // pub cache_created_at: String, } @@ -157,6 +177,7 @@ pub struct Config { pub struct Context { pub region: Option, pub config: Option, + pub cache: Option, pub overwritten_region: Option, // --region option pub overwritten_table_name: Option, // --table option pub output: Option, @@ -173,8 +194,11 @@ impl Context { // if region is overwritten by --region comamnd, use it. if let Some(ow_region) = &self.overwritten_region { return ow_region.to_owned(); }; - // next, if there's a region name in currently using table, use it. - if let Some(table) = self.config.as_ref().and_then(|x| x.clone().table ) { return region_from_str(Some(table.region)).unwrap(); }; + // next, if there's an `using_region` field in the config file, use it. + if let Some(using_region_name_in_config) = &self.config.to_owned().and_then(|x| x.using_region ) { + return region_from_str(Some(using_region_name_in_config.to_owned())) // Option + .expect("Region name in the config file is invalid."); + }; // otherwise, come down to "default region" of your environment. // e.g. region set via AWS CLI (check: $ aws configure get region), or environment variable `AWS_DEFAULT_REGION`. @@ -184,14 +208,27 @@ impl Context { } pub fn effective_table_name(&self) -> String { - // if table is overwritten by --table command, use it. + // if table is overwritten by --table option, use it. if let Some(ow_table_name) = &self.overwritten_table_name { return ow_table_name.to_owned(); }; - // otherwise, retrieve table name from config file. - return match self.config.as_ref().and_then(|x| x.table.to_owned() ) { - Some(table) => table.name, - // if both of data sources above are not available, raise error and exit the command. - None => { error!("{}", Messages::NoEffectiveTable); std::process::exit(1); }, - } + // otherwise, retrieve an `using_table` from config file. + return self.to_owned().config.and_then(|x| x.using_table ) + .unwrap_or_else(|| { // if both --option nor config file are not available, raise error and exit the command. + error!("{}", Messages::NoEffectiveTable); std::process::exit(1) + }); + } + + pub fn effective_cache_key(&self) -> String { + return format!("{}/{}", &self.effective_region().name(), &self.effective_table_name()); + } + + pub fn cached_using_table_schema(&self) -> Option { + let cached_tables: HashMap = match self.cache.to_owned().and_then(|c| c.tables) { + Some(cts) => cts, + None => return None, // return None for this "cached_using_table_schema" function + }; + let found_table_schema: Option<&TableSchema> = cached_tables.get(&self.effective_cache_key()); + // NOTE: HashMap's `get` returns a reference to the value / (&self, k: &Q) -> Option<&V> + return found_table_schema.map(|schema| schema.to_owned()); } pub fn with_region(mut self, ec2_region: &rusoto_ec2::Region) -> Self { @@ -266,9 +303,9 @@ fn region_dynamodb_local(port: u32) -> Region { } /// Loads dynein config file (YAML format) and return config struct as a result. -/// If it cannot find config file, create blank config. +/// Creates the file with default if the file couldn't be found. pub fn load_or_touch_config_file(first_try: bool) -> Result { - let path = retrieve_config_file_path(); + let path = retrieve_dynein_file_path(DyneinFileType::ConfigFile)?; debug!("Loading Config File: {}", path); match fs::read_to_string(&path) { @@ -280,37 +317,65 @@ pub fn load_or_touch_config_file(first_try: bool) -> Result { if !first_try { return Err(DyneinConfigError::from(e)) }; info!("Config file doesn't exist in the path, hence creating a blank file: {}", e); - touch_config_file()?; + let yaml_string = serde_yaml::to_string(&Config { ..Default::default() }).unwrap(); + fs::write(&retrieve_dynein_file_path(DyneinFileType::ConfigFile)?, yaml_string)?; load_or_touch_config_file(false) // set fisrt_try flag to false in order to avoid infinite loop. } } } -pub async fn use_table(cx: &Context) { - if let Some(tbl) = &cx.overwritten_table_name { - debug!("describing the table: {}", &tbl); - let region = cx.effective_region(); - let desc: TableDescription = describe_table_api(®ion, tbl.clone()).await; - let config = cx.config.clone().unwrap(); - save_table(region, config, desc); - } else { - bye(1, "ERROR: You have to specify a tabel to use by --table/-t option."); +/// Loads dynein cache file (YAML format) and return Cache struct as a result. +/// Creates the file with default if the file couldn't be found. +pub fn load_or_touch_cache_file(first_try: bool) -> Result { + let path = retrieve_dynein_file_path(DyneinFileType::CacheFile)?; + debug!("Loading Cache File: {}", path); + + match fs::read_to_string(&path) { + Ok(_str) => { + let cache: Cache = serde_yaml::from_str(&_str)?; + debug!("Loaded current cache: {:?}", cache); + Ok(cache) + }, + Err(e) => { + if !first_try { return Err(DyneinConfigError::from(e)) }; + info!("Config file doesn't exist in the path, hence creating a blank file: {}", e); + let yaml_string = serde_yaml::to_string(&Cache { ..Default::default() }).unwrap(); + fs::write(&retrieve_dynein_file_path(DyneinFileType::CacheFile)?, yaml_string)?; + load_or_touch_cache_file(false) // set fisrt_try flag to false in order to avoid infinite loop. + } } } -/// Physicall remove config file. -pub fn remove_config_file() -> std::io::Result<()> { - fs::remove_file(retrieve_config_file_path())?; +/// You can use a table in two syntaxes: +/// $ dy use --table mytable +/// or +/// $ dy use mytable +pub async fn use_table(cx: &Context, positional_arg_table_name: Option) -> Result<(), DyneinConfigError> { + // When context has "overwritten_table_name". i.e. you passed --table (-t) option. + // When you didn't pass --table option, check if you specified target table name directly, instead of --table option. + let target_table: Option = cx.clone().overwritten_table_name.or(positional_arg_table_name); + match target_table { + Some(tbl) => { + debug!("describing the table: {}", &tbl); + let region = cx.effective_region(); + let desc: TableDescription = describe_table_api(®ion, tbl.clone()).await; + + save_using_target(cx, desc)?; + println!("Now you're using the table '{}' ({}).", tbl, ®ion.name()); + }, + None => bye(1, "You have to specify a table. How to use (1). 'dy use --table mytable', or (2) 'dy use mytable'."), + }; + Ok(()) } -/// Create config file with blank (None) for each field. -pub fn touch_config_file() -> std::io::Result<()> { - let yaml_string = serde_yaml::to_string(&Config { table: None }).unwrap(); - fs::write(&retrieve_config_file_path(), yaml_string).expect("Could not write to file!"); +/// Physicall remove config and cache file. +pub fn remove_dynein_files() -> Result<(), DyneinConfigError> { + fs::remove_file(retrieve_dynein_file_path(DyneinFileType::ConfigFile)?)?; + fs::remove_file(retrieve_dynein_file_path(DyneinFileType::CacheFile)?)?; Ok(()) } @@ -363,7 +428,10 @@ pub async fn table_schema(cx: &Context) -> TableSchema { }, None => { // simply maps config data into TableSchema struct. debug!("current context {:#?}", cx); - return cx.config.clone().unwrap().table.unwrap_or_else(|| { + let cache: Cache = cx.clone().cache.expect("Cache should exist in context"); // can refactor here using and_then + let cached_tables: HashMap = cache.tables.expect("tables should exist in cache"); + let schema_from_cache: Option = cached_tables.get(&cx.effective_cache_key().clone()).map(|x| x.to_owned()); + return schema_from_cache.unwrap_or_else(|| { error!("{}", Messages::NoEffectiveTable); std::process::exit(1) }); }, @@ -429,32 +497,20 @@ pub fn bye(code: i32, msg: &str) { } -pub fn save_table(region: Region, config: Config, desc: TableDescription) { - let path = retrieve_config_file_path(); - - let mut new_config = config.clone(); - new_config.table = None; // reset existing table info - new_config.table = Some(TableSchema { - region: String::from(region.name()), - name: desc.table_name.clone().unwrap(), - pk: typed_key("HASH", &desc).expect("pk should exist"), - sk: typed_key("RANGE", &desc), - indexes: index_schemas(&desc), - mode: control::extract_mode(&desc.billing_mode_summary), - }); - - let yaml_string = serde_yaml::to_string(&new_config).expect("Failed to execute serde_yaml::to_string"); - fs::write(path, yaml_string).expect("Could not write to file!"); - debug!("Config Updated: {:#?}", &new_config); -} - - /* ================================================= Private functions ================================================= */ -fn retrieve_config_file_path () -> String { format!("{}/{}", retrieve_config_dir().unwrap(), CONFIG_FILE_NAME) } -fn retrieve_config_dir() -> Result { +fn retrieve_dynein_file_path (dft: DyneinFileType) -> Result { + let filename = match dft { + DyneinFileType::ConfigFile => CONFIG_FILE_NAME, + DyneinFileType::CacheFile => CACHE_FILE_NAME, + }; + + Ok(format!("{}/{}", retrieve_or_create_dynein_dir()?, filename)) +} + +fn retrieve_or_create_dynein_dir() -> Result { return match dirs::home_dir() { None => Err(DyneinConfigError::HomeDir), Some(home) => { @@ -469,6 +525,67 @@ fn retrieve_config_dir() -> Result { } +/// This function updates `using_region` and `using_table` in config.yml, +/// and at the same time inserts TableDescription of the target table into cache.yml. +fn save_using_target(cx: &Context, desc: TableDescription) -> Result<(), DyneinConfigError> { + let table_name: String = desc.table_name.clone().expect("desc should have table name"); + + // retrieve current config from Context and update "using target". + let mut config = cx.clone().config.expect("cx should have config").clone(); + config.using_region = Some(String::from(cx.effective_region().name())); + config.using_table = Some(table_name.clone()); + debug!("config file will be updated with: {:?}", &config); + + // write to config file + let config_yaml_string = serde_yaml::to_string(&config)?; + fs::write(retrieve_dynein_file_path(DyneinFileType::ConfigFile)?, config_yaml_string)?; + + // save target table info into cache. + insert_to_table_cache(&cx, desc)?; + + Ok(()) +} + + +/// Inserts specified table description into cache file. +pub fn insert_to_table_cache(cx: &Context, desc: TableDescription) -> Result<(), DyneinConfigError> { + let table_name: String = desc.table_name.clone().expect("desc should have table name"); + let region: Region = cx.effective_region(); + debug!("Under the region '{}', trying to save table schema of '{}'", ®ion.name(), &table_name); + + // retrieve current cache from Context and update target table desc. + // key to save the table desc is "/" -- e.g. "us-west-2/app_data" + let mut cache: Cache = cx.clone().cache.expect("cx should have cache"); + let cache_key = format!("{}/{}", region.name(), table_name.clone()); + + let mut table_schema_hashmap: HashMap = match cache.tables { + Some(ts) => ts, + None => HashMap::::new(), + }; + debug!("table schema cache before insert: {:#?}", table_schema_hashmap); + + table_schema_hashmap.insert( + cache_key, + TableSchema { + region: String::from(region.name()), + name: table_name.clone(), + pk: typed_key("HASH", &desc).expect("pk should exist"), + sk: typed_key("RANGE", &desc), + indexes: index_schemas(&desc), + mode: control::extract_mode(&desc.billing_mode_summary), + } + ); + cache.tables = Some(table_schema_hashmap); + + // write to cache file + let cache_yaml_string = serde_yaml::to_string(&cache)?; + debug!("this YAML will be written to the cache file: {:#?}", &cache_yaml_string); + fs::write(retrieve_dynein_file_path(DyneinFileType::CacheFile)?, cache_yaml_string)?; + + Ok(()) +} + + /* ================================================= Unit Tests ================================================= */ @@ -484,6 +601,7 @@ mod tests { let cx1 = Context { region: None, config: None, + cache: None, overwritten_region: None, overwritten_table_name: None, output: None, @@ -494,15 +612,10 @@ mod tests { let cx2 = Context { region: None, config: Some(Config { - table: Some(TableSchema { - region: String::from("ap-northeast-1"), - name: String::from("cfgtbl"), - pk: Key { name: String::from("pk"), kind: KeyType::S }, - sk: None, - indexes: None, - mode: control::Mode::OnDemand, - }), + using_region: Some(String::from("ap-northeast-1")), + using_table: Some(String::from("cfgtbl")), }), + cache: None, overwritten_region: None, overwritten_table_name: None, output: None, diff --git a/src/bootstrap.rs b/src/bootstrap.rs index cbb44fd4..c6c8aedb 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -259,7 +259,7 @@ https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AppendixSampleT println!(" $ dy --region {} scan --table Thread", &cx.effective_region().name()); println!(" $ dy --region {} use --table Thread", &cx.effective_region().name()); println!(" $ dy scan"); - println!("\nAfter you 'use' a table like above, dynein assume you're using the same region & table, which info is stored at ~/.dynein/config.yml"); + println!("\nAfter you 'use' a table like above, dynein assume you're using the same region & table, which info is stored at ~/.dynein/config.yml and ~/.dynein/cache.yml"); println!("Let's move on with the '{}' region you've just 'use'd...", &cx.effective_region().name()); println!(" $ dy scan --table Forum"); println!(" $ dy scan -t ProductCatalog"); diff --git a/src/cmd.rs b/src/cmd.rs index fcab272c..1eef1729 100644 --- a/src/cmd.rs +++ b/src/cmd.rs @@ -24,7 +24,7 @@ use structopt::StructOpt; const ABOUT_DYNEIN: &'static str = "\ dynein is a command line tool to interact with DynamoDB tables/data using concise interface.\n\ -dynein looks for a config file under $HOME/.dynein directory."; +dynein looks for config files under $HOME/.dynein/ directory."; #[derive(StructOpt, Debug)] #[structopt(name = "dynein", about = ABOUT_DYNEIN)] @@ -32,12 +32,13 @@ pub struct Dynein { #[structopt(subcommand)] pub child: Sub, - /// The region to use. When using DynamodB Local, `--region local` + /// The region to use (e.g. --region us-east-1). When using DynamodB Local, use `--region local`. /// You can use --region option in both top-level and subcommand-level. #[structopt(short, long, global = true)] pub region: Option, - /// Target table. By executing `$ dy use -t
` you can omit --table on every command. You can use --table option in both top-level and subcommand-level. + /// Target table of the operation. You can use --table option in both top-level and subcommand-level. + /// You can store table schema locally by executing `$ dy use`, after that you need not to specify --table on every command. #[structopt(short, long, global = true)] pub table: Option, } @@ -242,14 +243,17 @@ pub enum Sub { Dynein utility commands ================================================= */ - /// Switch target table context. You can overwrite the context with --table. + /// Switch target table context. After you use the command you don't need to specify table every time, but you may overwrite the target table with --table (-t) option. /// - /// When you execute `use` dynein stores table information in ~/.dynein/ directory. - /// Table information would be retrieved via DescribeTable API. + /// When you execute `use`, dynein retrieves table schema info via DescribeTable API + /// and stores it in ~/.dynein/ directory. #[structopt()] - Use { }, + Use { + /// Target table name to use. Optionally you may specify the target table by --table (-t) option. + target_table_to_use: Option + }, - /// Manage configuration file (~/.dynein/config.yml) from command line + /// Manage configuration files (config.yml and cache.yml) from command line #[structopt()] Config { #[structopt(subcommand)] @@ -459,11 +463,11 @@ pub enum DeleteSub { #[derive(StructOpt, Debug, Serialize, Deserialize)] pub enum ConfigSub { - /// Show all configuration in the config file (`~/.dynein/config.yml`). + /// Show all configuration in config (config.yml) and cache (cache.yml) files. #[structopt(aliases = &["show", "current-context"])] // for now, as config content is not so large, showing current context == dump all config. Dump, - /// Reset all configuration. The config file (`~/.dynein/config.yml`) would be initialized. + /// Reset all dynein configuration in the `~/.dynein/` directory. This command initializes dynein related files only and won't remove your data stored in DynamoDB tables. #[structopt()] Clear, } diff --git a/src/control.rs b/src/control.rs index 882e7c3e..e40ea0a6 100644 --- a/src/control.rs +++ b/src/control.rs @@ -101,7 +101,14 @@ pub async fn list_tables_all_regions(cx: app::Context) { let input: DescribeRegionsRequest = DescribeRegionsRequest { ..Default::default() }; match ec2.describe_regions(input).await { Err(e) => { error!("{}", e.to_string()); std::process::exit(1); }, - Ok(res) => join_all(res.regions.unwrap().iter().map(|r| list_tables(cx.clone().with_region(r)) )).await, + Ok(res) => { + join_all( + res.regions.expect("regions should exist") // Vec + .iter().map(|r| + list_tables(cx.clone().with_region(r)) + ) + ).await; + }, }; } @@ -109,12 +116,13 @@ pub async fn list_tables_all_regions(cx: app::Context) { pub async fn list_tables(cx: app::Context) { let table_names = list_tables_api(cx.clone()).await; - println!("DynamoDB tables in region: {}", &cx.effective_region().name()); + println!("DynamoDB tables in region: {}", cx.effective_region().name()); if table_names.len() == 0 { return println!(" No table in this region."); } - if let Some(table_in_config) = cx.clone().config.and_then(|x| x.table) { + // if let Some(table_in_config) = cx.clone().config.and_then(|x| x.table) { + if let Some(table_in_config) = cx.clone().cached_using_table_schema() { for table_name in table_names { - if cx.effective_region().name() == table_in_config.region && table_name == table_in_config.name { + if cx.clone().effective_region().name() == table_in_config.region && table_name == table_in_config.name { println!("* {}", table_name); } else { println!(" {}", table_name); @@ -128,6 +136,7 @@ pub async fn list_tables(cx: app::Context) { /// Executed when you call `$ dy desc --all-tables`. +/// Note that `describe_table` function calls are executed in parallel (async + join_all). pub async fn describe_all_tables(cx: app::Context) { let table_names = list_tables_api(cx.clone()).await; join_all(table_names.iter().map(|t| describe_table(cx.clone().with_table(t)) )).await; @@ -141,8 +150,12 @@ pub async fn describe_table(cx: app::Context) { let desc: TableDescription = app::describe_table_api(&cx.effective_region(), cx.effective_table_name()).await; debug!("Retrieved table to describe is: '{}' table in '{}' region.", &desc.clone().table_name.unwrap(), &cx.effective_region().name()); - // update table information stored in config dir - app::save_table(cx.effective_region(), app::Config { table: None }, desc.clone()); + // save described table info into cache for future use. + // Note that when this functiono is called from describe_all_tables, not all tables would be cached as calls are parallel. + match app::insert_to_table_cache(&cx, desc.clone()) { + Ok(_) => { debug!("Described table schema was written to the cache file.") }, + Err(e) => println!("Failed to write table schema to the cache with follwoing error: {:?}", e), + }; match cx.clone().output.as_ref().map(|x| x.as_str() ) { None | Some("yaml") => print_table_description(cx.effective_region(), desc), diff --git a/src/main.rs b/src/main.rs index 7c273d9f..0a6ee353 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,13 +47,12 @@ async fn main() -> Result<(), Box> { let c = cmd::initialize_from_args(); debug!("Command details: {:?}", c); - let config = app::load_or_touch_config_file(true)?; - // when --region , use the region. when --region local, use DynamoDB local. // --region/--table option can be passed as a top-level or subcommand-level (i.e. global). let mut context = app::Context { region: None, - config: Some(config), + config: Some(app::load_or_touch_config_file(true)?), + cache: Some(app::load_or_touch_cache_file(true)?), overwritten_region: app::region_from_str(c.region), overwritten_table_name: c.table, output: None, @@ -110,10 +109,13 @@ async fn main() -> Result<(), Box> { if all_tables { control::describe_all_tables(context).await } else { control::describe_table(context).await } }, - cmd::Sub::Use { } => app::use_table(&context).await, + cmd::Sub::Use { target_table_to_use } => app::use_table(&context, target_table_to_use).await?, cmd::Sub::Config { grandchild } => match grandchild { - cmd::ConfigSub::Dump => println!("{}", serde_yaml::to_string(&app::load_or_touch_config_file(true)?)?), - cmd::ConfigSub::Clear => app::remove_config_file().and_then(|()| app::touch_config_file())?, + cmd::ConfigSub::Dump => { + println!("{}", serde_yaml::to_string(&app::load_or_touch_cache_file(true)?)?); + println!("{}", serde_yaml::to_string(&app::load_or_touch_config_file(true)?)?); + }, + cmd::ConfigSub::Clear => app::remove_dynein_files()?, }, cmd::Sub::Bootstrap { list, sample } => { From 9412c1cf72720c73e3043673b40f1902c96cd14d Mon Sep 17 00:00:00 2001 From: Takuya Hashimoto Date: Tue, 17 Nov 2020 23:41:30 +0900 Subject: [PATCH 2/2] Combine public and private functions in different places --- src/app.rs | 97 +++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/src/app.rs b/src/app.rs index f7521839..bd961b20 100644 --- a/src/app.rs +++ b/src/app.rs @@ -293,15 +293,6 @@ pub fn region_from_str(s: Option) -> Option { } -fn region_dynamodb_local(port: u32) -> Region { - let endpoint_url = format!("http://localhost:{}", port); - debug!("setting DynamoDB Local '{}' as target region.", &endpoint_url); - return Region::Custom { - name: "local".to_owned(), - endpoint: endpoint_url.to_owned(), - }; -} - /// Loads dynein config file (YAML format) and return config struct as a result. /// Creates the file with default if the file couldn't be found. pub fn load_or_touch_config_file(first_try: bool) -> Result { @@ -372,6 +363,45 @@ pub async fn use_table(cx: &Context, positional_arg_table_name: Option) } +/// Inserts specified table description into cache file. +pub fn insert_to_table_cache(cx: &Context, desc: TableDescription) -> Result<(), DyneinConfigError> { + let table_name: String = desc.table_name.clone().expect("desc should have table name"); + let region: Region = cx.effective_region(); + debug!("Under the region '{}', trying to save table schema of '{}'", ®ion.name(), &table_name); + + // retrieve current cache from Context and update target table desc. + // key to save the table desc is "/" -- e.g. "us-west-2/app_data" + let mut cache: Cache = cx.clone().cache.expect("cx should have cache"); + let cache_key = format!("{}/{}", region.name(), table_name.clone()); + + let mut table_schema_hashmap: HashMap = match cache.tables { + Some(ts) => ts, + None => HashMap::::new(), + }; + debug!("table schema cache before insert: {:#?}", table_schema_hashmap); + + table_schema_hashmap.insert( + cache_key, + TableSchema { + region: String::from(region.name()), + name: table_name.clone(), + pk: typed_key("HASH", &desc).expect("pk should exist"), + sk: typed_key("RANGE", &desc), + indexes: index_schemas(&desc), + mode: control::extract_mode(&desc.billing_mode_summary), + } + ); + cache.tables = Some(table_schema_hashmap); + + // write to cache file + let cache_yaml_string = serde_yaml::to_string(&cache)?; + debug!("this YAML will be written to the cache file: {:#?}", &cache_yaml_string); + fs::write(retrieve_dynein_file_path(DyneinFileType::CacheFile)?, cache_yaml_string)?; + + Ok(()) +} + + /// Physicall remove config and cache file. pub fn remove_dynein_files() -> Result<(), DyneinConfigError> { fs::remove_file(retrieve_dynein_file_path(DyneinFileType::ConfigFile)?)?; @@ -501,6 +531,16 @@ pub fn bye(code: i32, msg: &str) { Private functions ================================================= */ +fn region_dynamodb_local(port: u32) -> Region { + let endpoint_url = format!("http://localhost:{}", port); + debug!("setting DynamoDB Local '{}' as target region.", &endpoint_url); + return Region::Custom { + name: "local".to_owned(), + endpoint: endpoint_url.to_owned(), + }; +} + + fn retrieve_dynein_file_path (dft: DyneinFileType) -> Result { let filename = match dft { DyneinFileType::ConfigFile => CONFIG_FILE_NAME, @@ -547,45 +587,6 @@ fn save_using_target(cx: &Context, desc: TableDescription) -> Result<(), DyneinC } -/// Inserts specified table description into cache file. -pub fn insert_to_table_cache(cx: &Context, desc: TableDescription) -> Result<(), DyneinConfigError> { - let table_name: String = desc.table_name.clone().expect("desc should have table name"); - let region: Region = cx.effective_region(); - debug!("Under the region '{}', trying to save table schema of '{}'", ®ion.name(), &table_name); - - // retrieve current cache from Context and update target table desc. - // key to save the table desc is "/" -- e.g. "us-west-2/app_data" - let mut cache: Cache = cx.clone().cache.expect("cx should have cache"); - let cache_key = format!("{}/{}", region.name(), table_name.clone()); - - let mut table_schema_hashmap: HashMap = match cache.tables { - Some(ts) => ts, - None => HashMap::::new(), - }; - debug!("table schema cache before insert: {:#?}", table_schema_hashmap); - - table_schema_hashmap.insert( - cache_key, - TableSchema { - region: String::from(region.name()), - name: table_name.clone(), - pk: typed_key("HASH", &desc).expect("pk should exist"), - sk: typed_key("RANGE", &desc), - indexes: index_schemas(&desc), - mode: control::extract_mode(&desc.billing_mode_summary), - } - ); - cache.tables = Some(table_schema_hashmap); - - // write to cache file - let cache_yaml_string = serde_yaml::to_string(&cache)?; - debug!("this YAML will be written to the cache file: {:#?}", &cache_yaml_string); - fs::write(retrieve_dynein_file_path(DyneinFileType::CacheFile)?, cache_yaml_string)?; - - Ok(()) -} - - /* ================================================= Unit Tests ================================================= */