From 320020dc841e957238c6c8fd3207e16ee58b8a7f Mon Sep 17 00:00:00 2001 From: Sergii Mikhtoniuk Date: Mon, 7 Oct 2024 17:26:22 -0700 Subject: [PATCH 01/16] Add query commitments example --- examples/query-commitments/README.md | 17 ++ examples/query-commitments/example.py | 266 +++++++++++++++++++++ examples/query-commitments/requirements.in | 4 + examples/reth-vs-snp500/init-odf-all.sh | 23 ++ examples/reth-vs-snp500/init-odf.sh | 17 ++ 5 files changed, 327 insertions(+) create mode 100644 examples/query-commitments/README.md create mode 100755 examples/query-commitments/example.py create mode 100644 examples/query-commitments/requirements.in create mode 100755 examples/reth-vs-snp500/init-odf-all.sh create mode 100755 examples/reth-vs-snp500/init-odf.sh diff --git a/examples/query-commitments/README.md b/examples/query-commitments/README.md new file mode 100644 index 0000000000..1b83763ae4 --- /dev/null +++ b/examples/query-commitments/README.md @@ -0,0 +1,17 @@ +# Examples of Verifiable Batch Queries +Query proofs allow you to hold any ODF node forever accountable for the result it provided you, no matter how much time had passed. + +See [batch query commitments documentation](https://docs.kamu.dev/node/commitments/) and [REST API reference](https://docs.kamu.dev/node/protocols/rest-api/) for the overview of this mechanism. + +The included script illustrates: +- how to query data and receive a cryptographic proof +- how to validate commitment consistency on the client side +- and how to ask another node to verify the commitment by reproducing the query. + +Running: +```sh +pip install -r requirements.in +python ./example.py --node-url https://node.example.com --private-key +``` + +Private key is used to show examples of failed verification (by forging a signature). If you don't provide it - those cases will be skipped. diff --git a/examples/query-commitments/example.py b/examples/query-commitments/example.py new file mode 100755 index 0000000000..5a2b78e6ea --- /dev/null +++ b/examples/query-commitments/example.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +import argparse +import copy +import canonicaljson +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +import hashlib +import requests +import json +import base64 +import base58 + + +def decode_multibase(multibase): + prefix, value = multibase[0], multibase[1:] + if prefix == 'u': + b64_urlsafe_nopad = value + b64_urlsafe = b64_urlsafe_nopad + "=" * ((4 - len(b64_urlsafe_nopad) % 4) % 4) + return base64.urlsafe_b64decode(b64_urlsafe) + elif prefix == 'f': + return bytes.fromhex(value) + elif prefix == 'z': + return base58.b58decode(value) + else: + raise("Malformed multibase value") + + +def encode_multibase_base64_urlsafe_nopad(bin): + return 'u' + base64.urlsafe_b64encode(bin).decode('utf-8').rstrip("=") + + +def multihash_sha3_256_multibase_base16(bin): + return 'f1620' + hashlib.sha3_256(bin).hexdigest() + + +def public_key_from_did(did): + assert did.startswith('did:key:') + multicodec = decode_multibase(did[len('did:key:'):]) + assert multicodec[0:1].hex() == 'ed' # 0xed is multicodec value for Ed25519Pub + return Ed25519PublicKey.from_public_bytes(multicodec[2:]) + +def main(args): + # Query data + resp = requests.get( + args.node_url + "/query", + params=dict( + query=args.query, + include="proof", + ) + ) + resp.raise_for_status() + resp_data = resp.json() + + print(">>> Node's response:") + print(json.dumps(resp_data, indent=2)) + print() + print() + + + # Verify commitment consistency + # This should always be done by the client after receiving a proof to ensure its disputable + assert resp_data["commitment"]["inputHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["input"]) + ) + assert resp_data["commitment"]["outputHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["output"]) + ) + assert resp_data["commitment"]["subQueriesHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["subQueries"]) + ) + + signature = decode_multibase(resp_data["proof"]["proofValue"]) + public_key = public_key_from_did(resp_data["proof"]["verificationMethod"]) + public_key.verify(signature, canonicaljson.encode_canonical_json(resp_data["commitment"])) + print("Commitment is consistent!") + + + print(">>> Commitment:") + commitment = resp_data.copy() + del commitment["output"] + print(json.dumps(commitment, indent=2)) + print() + print() + + + # Remote validation through reproducibility + print(">>> Verifying original commitment:") + resp = requests.post( + args.node_url + "/verify", + json=commitment + ) + resp.raise_for_status() + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == True + + + # Invalid request: input hash + print(">>> Simulating invalid request (input hash):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["inputHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::InputHash" + + + # Invalid request: subQueries hash + print(">>> Simulating invalid request (subQueries hash):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["subQueriesHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::SubQueriesHash" + + + # Invalid request: bad signature + print(">>> Simulating invalid request (bad signature):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["proof"]["proofValue"] = "uZbm7fFcWc4l6iyvaKe_txdKntL3h3kvsGHOaKIbPV6c42PH1VnSmpYHMopv4TU68syzgoEdcS26AvpkSQb9dBQ" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::BadSignature" + + + if args.private_key is None: + print("Private key is not provided - skipping tests that require signature forging") + return + + private_key = Ed25519PrivateKey.from_private_bytes(decode_multibase(args.private_key)) + + + # Cannot reproduce the query: output mismatch + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (output mismatch):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["outputHash"] = "f1620ff7f5beaf16900218a3ac4aae82cdccf764816986c7c739c716cf7dc03112a2d" + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::OutputMismatch" + + + # Cannot reproduce the query: dataset is missing + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (dataset is missing):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["input"]["datasets"][0]["id"] = invalid_commitment["input"]["datasets"][0]["id"][:-4] + "beef" + invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json( + invalid_commitment["input"] + ) + ) + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::DatasetNotFound" + + + # Cannot reproduce the query: block is missing + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (block is missing):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["input"]["datasets"][0]["blockHash"] = invalid_commitment["input"]["datasets"][0]["blockHash"][:-4] + "beef" + invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json( + invalid_commitment["input"] + ) + ) + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::DatasetBlockNotFound" + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--node-url', required=True) + parser.add_argument('--private-key', required=False) + parser.add_argument('--query', default='select block_hash, to from "kamu/net.rocketpool.reth.tokens-minted" order by offset desc limit 1') + args = parser.parse_args() + main(args) \ No newline at end of file diff --git a/examples/query-commitments/requirements.in b/examples/query-commitments/requirements.in new file mode 100644 index 0000000000..ab209766b2 --- /dev/null +++ b/examples/query-commitments/requirements.in @@ -0,0 +1,4 @@ +base58 +canonicaljson +cryptography +requests \ No newline at end of file diff --git a/examples/reth-vs-snp500/init-odf-all.sh b/examples/reth-vs-snp500/init-odf-all.sh new file mode 100755 index 0000000000..1739ff3877 --- /dev/null +++ b/examples/reth-vs-snp500/init-odf-all.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +NODE_URL="odf+https://node.demo.kamu.dev/kamu/" + +kamu init || true + +# Root +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" +kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" +kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" + +kamu pull "${NODE_URL}account.transactions" +kamu pull "${NODE_URL}account.tokens.transfers" + +# Deriv +kamu pull "${NODE_URL}net.rocketpool.reth.mint-burn" +kamu pull "${NODE_URL}account.tokens.portfolio" +kamu pull "${NODE_URL}account.tokens.portfolio.market-value" +kamu pull "${NODE_URL}account.tokens.portfolio.usd" +kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.market-value" +kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.portfolio" diff --git a/examples/reth-vs-snp500/init-odf.sh b/examples/reth-vs-snp500/init-odf.sh new file mode 100755 index 0000000000..abec551cce --- /dev/null +++ b/examples/reth-vs-snp500/init-odf.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +NODE_URL="odf+https://node.demo.kamu.dev/kamu/" + +kamu init || true + +# Root +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" +kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" +kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" + +kamu pull "${NODE_URL}account.transactions" +kamu pull "${NODE_URL}account.tokens.transfers" + +kamu add -r . From 518474852f78fdf42172d885f510537008dac0c3 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Tue, 8 Oct 2024 12:59:03 +0300 Subject: [PATCH 02/16] Release (patch) `0.204.5`: fix `--yes / -y` flag: fixed when working from a TTY (#881) * Interact: fix if is_tty * CHANGELOG.md: update * Release (patch): 0.204.5 * Updated yanked crate (futures-util to 0.3.31) --- CHANGELOG.md | 4 +- Cargo.lock | 174 ++++++++++++++--------------- Cargo.toml | 138 +++++++++++------------ LICENSE.txt | 2 +- src/app/cli/src/output/interact.rs | 20 +++- 5 files changed, 174 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e832ef63be..16c7413421 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Recommendation: for ease of reading, use the following order: - Fixed --> -## [Unreleased] +## [0.204.5] - 2024-10-08 ### Added - Postgres implementation for dataset entry and account Re-BAC repositories ### Changed @@ -24,6 +24,8 @@ Recommendation: for ease of reading, use the following order: - Simplified error handling code in repositories - Hidden part of the test code behind the feature gate - Updated our crate dependencies so they can be built in isolation +### Fixed +- `--yes / -y` flag: fixed when working from a TTY ## [0.204.4] - 2024-09-30 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 7f63ed6e32..fe8e7297b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1311,7 +1311,7 @@ dependencies = [ [[package]] name = "async-utils" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", ] @@ -2449,7 +2449,7 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "cfg-if", @@ -2879,7 +2879,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "aws-config", @@ -2903,7 +2903,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.204.4" +version = "0.204.5" dependencies = [ "quote", "syn 2.0.79", @@ -3747,7 +3747,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.204.4" +version = "0.204.5" [[package]] name = "env_filter" @@ -3816,7 +3816,7 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.204.4" +version = "0.204.5" dependencies = [ "quote", "syn 2.0.79", @@ -4012,9 +4012,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -4022,9 +4022,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" @@ -4050,15 +4050,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -4067,15 +4067,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -4085,9 +4085,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -4500,7 +4500,7 @@ dependencies = [ [[package]] name = "http-common" -version = "0.204.4" +version = "0.204.5" dependencies = [ "axum", "http 1.1.0", @@ -4799,7 +4799,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "init-on-startup" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "database-common", @@ -4842,7 +4842,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.204.4" +version = "0.204.5" dependencies = [ "thiserror", ] @@ -4990,7 +4990,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.204.4" +version = "0.204.5" dependencies = [ "alloy", "async-recursion", @@ -5078,7 +5078,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "base32", @@ -5104,7 +5104,7 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5125,7 +5125,7 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5146,7 +5146,7 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5167,7 +5167,7 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "argon2", "chrono", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.204.4" +version = "0.204.5" dependencies = [ "argon2", "async-trait", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5231,7 +5231,7 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "dill", @@ -5253,7 +5253,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.204.4" +version = "0.204.5" dependencies = [ "arrow-flight", "async-trait", @@ -5276,7 +5276,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-graphql", "async-trait", @@ -5327,7 +5327,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5393,7 +5393,7 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5412,7 +5412,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.204.4" +version = "0.204.5" dependencies = [ "axum", "chrono", @@ -5448,7 +5448,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "internal-error", @@ -5460,7 +5460,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "database-common-macros", @@ -5474,7 +5474,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "database-common", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "dill", "kamu-auth-rebac", @@ -5500,7 +5500,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "dill", @@ -5519,7 +5519,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "database-common", @@ -5536,7 +5536,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.204.4" +version = "0.204.5" dependencies = [ "arrow-flight", "async-graphql", @@ -5657,7 +5657,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5678,7 +5678,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.204.4" +version = "0.204.5" dependencies = [ "quote", "syn 2.0.79", @@ -5686,7 +5686,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5699,7 +5699,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.204.4" +version = "0.204.5" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5713,7 +5713,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5727,7 +5727,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "indoc 2.0.5", @@ -5744,7 +5744,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5758,7 +5758,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.204.4" +version = "0.204.5" dependencies = [ "assert_cmd", "async-trait", @@ -5774,7 +5774,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.204.4" +version = "0.204.5" dependencies = [ "arrow", "arrow-digest", @@ -5830,7 +5830,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.204.4" +version = "0.204.5" dependencies = [ "arrow", "async-trait", @@ -5854,7 +5854,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.204.4" +version = "0.204.5" dependencies = [ "aes-gcm", "async-trait", @@ -5874,7 +5874,7 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5897,7 +5897,7 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5920,7 +5920,7 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "database-common", @@ -5934,7 +5934,7 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5965,7 +5965,7 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -5988,7 +5988,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6017,7 +6017,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6047,7 +6047,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6072,7 +6072,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "database-common", @@ -6085,7 +6085,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6190,7 +6190,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6209,7 +6209,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6232,7 +6232,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "database-common", @@ -6246,7 +6246,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6268,7 +6268,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "clap", @@ -6283,7 +6283,7 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.204.4" +version = "0.204.5" dependencies = [ "chrono", "database-common", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-stream", "async-trait", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -6909,7 +6909,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.204.4" +version = "0.204.5" dependencies = [ "base64 0.22.1", "bs58", @@ -7234,7 +7234,7 @@ dependencies = [ [[package]] name = "observability" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "axum", @@ -7293,7 +7293,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.204.4" +version = "0.204.5" dependencies = [ "arrow", "base64 0.22.1", @@ -8253,7 +8253,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.204.4" +version = "0.204.5" dependencies = [ "rand", ] @@ -9824,7 +9824,7 @@ dependencies = [ [[package]] name = "time-source" -version = "0.204.4" +version = "0.204.5" dependencies = [ "async-trait", "chrono", @@ -10238,7 +10238,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.204.4" +version = "0.204.5" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index a5ddbd7b05..3e025cf609 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,95 +92,95 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.204.4", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.204.5", path = "src/app/cli", default-features = false } # Utils -async-utils = { version = "0.204.4", path = "src/utils/async-utils", default-features = false } -container-runtime = { version = "0.204.4", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.204.4", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.204.4", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.204.4", path = "src/utils/enum-variants", default-features = false } -event-sourcing = { version = "0.204.4", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.204.4", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.204.4", path = "src/utils/http-common", default-features = false } -init-on-startup = { version = "0.204.4", path = "src/utils/init-on-startup", default-features = false } -internal-error = { version = "0.204.4", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.204.4", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.204.4", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.204.4", path = "src/utils/datafusion-cli", default-features = false } -messaging-outbox = { version = "0.204.4", path = "src/utils/messaging-outbox", default-features = false } -multiformats = { version = "0.204.4", path = "src/utils/multiformats", default-features = false } -observability = { version = "0.204.4", path = "src/utils/observability", default-features = false } -random-names = { version = "0.204.4", path = "src/utils/random-names", default-features = false } -time-source = { version = "0.204.4", path = "src/utils/time-source", default-features = false } -tracing-perfetto = { version = "0.204.4", path = "src/utils/tracing-perfetto", default-features = false } +async-utils = { version = "0.204.5", path = "src/utils/async-utils", default-features = false } +container-runtime = { version = "0.204.5", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.204.5", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.204.5", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.204.5", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.204.5", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.204.5", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.204.5", path = "src/utils/http-common", default-features = false } +init-on-startup = { version = "0.204.5", path = "src/utils/init-on-startup", default-features = false } +internal-error = { version = "0.204.5", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.204.5", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.204.5", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.204.5", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.204.5", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.204.5", path = "src/utils/multiformats", default-features = false } +observability = { version = "0.204.5", path = "src/utils/observability", default-features = false } +random-names = { version = "0.204.5", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.204.5", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.204.5", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.204.4", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.204.4", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.204.4", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.204.4", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.204.4", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.204.4", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.204.4", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.204.5", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.204.5", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.204.5", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.204.5", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.204.5", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.204.5", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.204.5", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.204.4", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.204.4", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.204.4", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.204.4", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.204.4", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.204.5", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.204.5", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.204.5", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.204.5", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.204.5", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.204.4", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.204.4", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.204.5", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.204.5", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.204.4", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.204.4", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.204.4", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.204.4", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.204.5", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.204.5", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.204.5", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.204.5", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.204.4", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.204.4", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.204.4", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.204.4", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.204.4", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.204.5", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.204.5", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.204.5", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.204.5", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.204.5", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.204.4", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.204.4", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.204.4", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.204.4", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.204.5", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.204.5", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.204.5", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.204.5", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.204.4", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.204.4", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.204.4", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.204.4", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.204.5", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.204.5", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.204.5", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.204.5", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.204.4", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.204.4", path = "src/infra/auth-rebac/repo-tests", default-features = false } -kamu-auth-rebac-postgres = { version = "0.204.4", path = "src/infra/auth-rebac/postgres", default-features = false } -kamu-auth-rebac-sqlite = { version = "0.204.4", path = "src/infra/auth-rebac/sqlite", default-features = false } +kamu-auth-rebac-inmem = { version = "0.204.5", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.204.5", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-postgres = { version = "0.204.5", path = "src/infra/auth-rebac/postgres", default-features = false } +kamu-auth-rebac-sqlite = { version = "0.204.5", path = "src/infra/auth-rebac/sqlite", default-features = false } ## Outbox -kamu-messaging-outbox-inmem = { version = "0.204.4", path = "src/infra/messaging-outbox/inmem", default-features = false } -kamu-messaging-outbox-postgres = { version = "0.204.4", path = "src/infra/messaging-outbox/postgres", default-features = false } -kamu-messaging-outbox-sqlite = { version = "0.204.4", path = "src/infra/messaging-outbox/sqlite", default-features = false } -kamu-messaging-outbox-repo-tests = { version = "0.204.4", path = "src/infra/messaging-outbox/repo-tests", default-features = false } +kamu-messaging-outbox-inmem = { version = "0.204.5", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.204.5", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.204.5", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.204.5", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.204.4", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.204.4", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.204.4", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.204.4", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.204.4", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.204.4", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.204.5", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.204.5", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.204.5", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.204.5", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.204.5", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.204.5", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.204.4", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.204.4", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.204.4", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.204.5", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.204.5", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.204.5", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.204.4" +version = "0.204.5" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index 5a0c90820d..7f5413248c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.204.4 +Licensed Work: Kamu CLI Version 0.204.5 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, diff --git a/src/app/cli/src/output/interact.rs b/src/app/cli/src/output/interact.rs index 13b4e4febf..94a892a4a4 100644 --- a/src/app/cli/src/output/interact.rs +++ b/src/app/cli/src/output/interact.rs @@ -9,6 +9,8 @@ use crate::CLIError; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[derive(Debug, Clone)] pub struct Interact { /// Don't ask user for confirmation and assume 'yes' @@ -26,17 +28,21 @@ impl Interact { pub fn require_confirmation(&self, prompt: impl std::fmt::Display) -> Result<(), CLIError> { use read_input::prelude::*; + // If there's confirmation, we don't need to ask anything of the user + if self.assume_yes { + return Ok(()); + } + let prompt = format!("{prompt}\nDo you wish to continue? [y/N]: "); + // If no data can be entered, we abort if !self.is_tty { - return if self.assume_yes { - Ok(()) - } else { - eprintln!("{prompt} Assuming 'no' because --yes flag was not provided"); - Err(CLIError::Aborted) - }; + eprintln!("{prompt} Assuming 'no' because --yes flag was not provided"); + + return Err(CLIError::Aborted); } + // In other cases, we ask until we get a valid answer let answer: String = input() .repeat_msg(prompt) .default("n".to_owned()) @@ -50,3 +56,5 @@ impl Interact { } } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From f060d0a445dda41ac8d5264ab577ca2933b9cb29 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Tue, 8 Oct 2024 16:01:52 +0300 Subject: [PATCH 03/16] CI: Fixes `kamu-base-with-data-mt` image builds (#885) * Images, kamu-base-with-data-mt: add "kamu" to predefined users * Images, kamu-base-with-data-mt: init-workspace.py use .kamuconfig * CHANGELOG.md: update --- CHANGELOG.md | 1 + .../kamu-base-with-data-mt/extra/.kamuconfig | 3 +++ .../kamu-base-with-data-mt/init-workspace.py | 21 ++++++++++++------- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c7413421..4fd5d8503f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Recommendation: for ease of reading, use the following order: - Updated our crate dependencies so they can be built in isolation ### Fixed - `--yes / -y` flag: fixed when working from a TTY +- CI: Fixes `kamu-base-with-data-mt` image builds ## [0.204.4] - 2024-09-30 ### Changed diff --git a/images/kamu-base-with-data-mt/extra/.kamuconfig b/images/kamu-base-with-data-mt/extra/.kamuconfig index 0462429714..68000e8f80 100644 --- a/images/kamu-base-with-data-mt/extra/.kamuconfig +++ b/images/kamu-base-with-data-mt/extra/.kamuconfig @@ -3,6 +3,9 @@ version: 1 content: users: predefined: + - accountName: kamu + isAdmin: true + avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4 - accountName: sh101-bowen avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png - accountName: sh102-gambier diff --git a/images/kamu-base-with-data-mt/init-workspace.py b/images/kamu-base-with-data-mt/init-workspace.py index 1d2b50a361..286c24ce44 100755 --- a/images/kamu-base-with-data-mt/init-workspace.py +++ b/images/kamu-base-with-data-mt/init-workspace.py @@ -1,48 +1,55 @@ #!/usr/bin/env python -import os -import sys +import shutil import subprocess +from pathlib import Path ############################################################################### +CURRENT_PATH = Path(__file__).resolve().parent S3_REPO_URL = "s3://datasets.kamu.dev/odf/v2/example-mt/" + ############################################################################### def s3_listdir(url): return [ line.strip().split(' ')[1] for line in subprocess.run( - f"aws s3 ls {url}", - shell=True, + f"aws s3 ls {url}", + shell=True, text=True, check=True, capture_output=True, ).stdout.splitlines() ] + def s3_cat(url): return subprocess.run( - f"aws s3 cp {url} -", - shell=True, + f"aws s3 cp {url} -", + shell=True, text=True, check=True, capture_output=True, ).stdout.strip() + ############################################################################### subprocess.run( - "kamu init --multi-tenant --exists-ok", + "kamu init --multi-tenant --exists-ok", shell=True, check=True, ) +shutil.copy(CURRENT_PATH / "extra/.kamuconfig", ".kamuconfig") + for did in s3_listdir(S3_REPO_URL): url = S3_REPO_URL + did alias = s3_cat(f"{S3_REPO_URL}{did}info/alias") account, name = alias.split('/', 1) + subprocess.run( f"kamu --account {account} pull --no-alias {url} --as {name}", shell=True, From ece9ad4bf476e3914a1a28258a312ff228e4f3de Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Thu, 10 Oct 2024 10:51:27 +0200 Subject: [PATCH 04/16] Add push pull tests --- .../common/src/kamu_api_server_client_ext.rs | 19 +- .../tests/test_smart_transfer_protocol.rs | 44 ++ src/e2e/app/cli/repo-tests/src/test_flow.rs | 8 +- .../src/test_smart_transfer_protocol.rs | 635 ++++++++++++++++-- .../src/kamu_cli_puppet_ext.rs | 14 + 5 files changed, 656 insertions(+), 64 deletions(-) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index aafe912e59..c08e2d964a 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -9,6 +9,7 @@ use async_trait::async_trait; use lazy_static::lazy_static; +use opendatafabric::AccountName; use reqwest::{Method, StatusCode}; use crate::{KamuApiServerClient, RequestBody}; @@ -114,7 +115,11 @@ pub trait KamuApiServerClientExt { async fn login_as_e2e_user(&self) -> AccessToken; async fn create_dataset(&self, dataset_snapshot_yaml: &str, token: &AccessToken) -> DatasetId; async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId; - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId; + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option<&AccountName>, + ) -> DatasetId; async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId; } @@ -201,13 +206,21 @@ impl KamuApiServerClientExt for KamuApiServerClient { .await } - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId { + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option<&AccountName>, + ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; + let mut base_path = "player-scores/ingest".to_string(); + if let Some(account_name) = account_name_maybe { + base_path = format!("{account_name}/{base_path}"); + } self.rest_api_call_assert( Some(token.clone()), Method::POST, - "player-scores/ingest", + base_path.as_str(), Some(RequestBody::NdJson( indoc::indoc!( r#" diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index dfa4094c22..0e7313b201 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -24,3 +24,47 @@ kamu_cli_run_api_server_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/test_flow.rs b/src/e2e/app/cli/repo-tests/src/test_flow.rs index 855724fc5c..8bfc35fec5 100644 --- a/src/e2e/app/cli/repo-tests/src/test_flow.rs +++ b/src/e2e/app/cli/repo-tests/src/test_flow.rs @@ -17,7 +17,7 @@ pub async fn test_get_dataset_list_flows(kamu_api_server_client: KamuApiServerCl let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -88,7 +88,7 @@ pub async fn test_dataset_all_flows_paused(kamu_api_server_client: KamuApiServer let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -147,7 +147,7 @@ pub async fn test_dataset_flows_initiators(kamu_api_server_client: KamuApiServer let token = kamu_api_server_client.login_as_kamu().await; let dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; // The query is almost identical to kamu-web-ui, for ease of later edits. @@ -228,7 +228,7 @@ pub async fn test_dataset_trigger_flow(kamu_api_server_client: KamuApiServerClie let token = kamu_api_server_client.login_as_kamu().await; let _root_dataset_id = kamu_api_server_client - .create_player_scores_dataset_with_data(&token) + .create_player_scores_dataset_with_data(&token, None) .await; let derivative_dataset_id = kamu_api_server_client.create_leaderboard(&token).await; diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 4de9a16638..356f133cec 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -7,13 +7,34 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use core::panic; + +use kamu::testing::MetadataFactory; use kamu_cli_e2e_common::{KamuApiServerClient, KamuApiServerClientExt}; +use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{ + AccountName, + AddPushSource, + DatasetAlias, + DatasetName, + DatasetSnapshot, + MergeStrategyLedger, + ReadStepNdJson, + SetTransform, + SetVocab, + SourceState, + Transform, + TransformInput, + TransformSql, +}; use reqwest::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; @@ -37,72 +58,52 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer // 2.1. Add the dataset { - let dataset_path = kamu_in_push_workspace - .workspace_path() - .join("player-scores.yaml"); - - std::fs::write( - dataset_path.clone(), - indoc::indoc!( - r#" - kind: DatasetSnapshot - version: 1 - content: - name: player-scores - kind: Root - metadata: - - kind: AddPushSource - sourceName: default - read: - kind: NdJson - schema: - - "match_time TIMESTAMP" - - "match_id BIGINT" - - "player_id STRING" - - "score BIGINT" - merge: - kind: Ledger - primaryKey: - - match_id - - player_id - - kind: SetVocab - eventTimeColumn: match_time - "# - ), - ) - .unwrap(); - kamu_in_push_workspace - .execute(["add", dataset_path.to_str().unwrap()]) - .await - .success(); + .add_dataset(DatasetSnapshot { + name: DatasetAlias::new(None, dataset_name.clone()), + kind: opendatafabric::DatasetKind::Root, + metadata: vec![ + AddPushSource { + source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), + read: ReadStepNdJson { + schema: Some(vec![ + "match_time TIMESTAMP".to_owned(), + "match_id BIGINT".to_owned(), + "player_id STRING".to_owned(), + "score BIGINT".to_owned(), + ]), + ..Default::default() + } + .into(), + preprocess: None, + merge: MergeStrategyLedger { + primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], + } + .into(), + } + .into(), + SetVocab { + event_time_column: Some("match_time".to_owned()), + ..Default::default() + } + .into(), + ], + }) + .await; } // 2.1. Ingest data to the dataset { - let dataset_data_path = kamu_in_push_workspace - .workspace_path() - .join("player-scores.data.ndjson"); - - std::fs::write( - dataset_data_path.clone(), - indoc::indoc!( - r#" + let data = indoc::indoc!( + r#" {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ), - ) - .unwrap(); + "#, + ); kamu_in_push_workspace - .execute([ - "ingest", - "player-scores", - dataset_data_path.to_str().unwrap(), - ]) - .await - .success(); + .ingest_data(&dataset_name, data) + .await; } // 2.2. Login to the API server @@ -140,3 +141,523 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("e2e-user/{dataset_name}").as_str()) + .unwrap() + .to_string() + }; + + // 2. Pushing the dataset to the API server + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2.1. Add the dataset + { + kamu_in_push_workspace + .add_dataset(DatasetSnapshot { + name: DatasetAlias::new(None, dataset_name.clone()), + kind: opendatafabric::DatasetKind::Root, + metadata: vec![ + AddPushSource { + source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), + read: ReadStepNdJson { + schema: Some(vec![ + "match_time TIMESTAMP".to_owned(), + "match_id BIGINT".to_owned(), + "player_id STRING".to_owned(), + "score BIGINT".to_owned(), + ]), + ..Default::default() + } + .into(), + preprocess: None, + merge: MergeStrategyLedger { + primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], + } + .into(), + } + .into(), + SetVocab { + event_time_column: Some("match_time".to_owned()), + ..Default::default() + } + .into(), + ], + }) + .await; + } + + // 2.1. Ingest data to the dataset + { + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Initial dataset push + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ]) + .await + .success(); + + // Hard compact dataset + kamu_in_push_workspace + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + // Should fail without force flag + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ]) + .await + .failure(); + + // Should successfully push with force flag + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--force", + ]) + .await + .success(); + } + + // 3. Pulling the dataset from the API server + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Call with no-alias flag to avoid remote ingest checking in next step + kamu_in_pull_workspace + .execute([ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ]) + .await + .success(); + + // Ingest data in pulled dataset + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Chuck", "score": 90} + "#, + ); + + kamu_in_pull_workspace + .ingest_data(&dataset_name, data) + .await; + + // Should fail without force flag + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) + .await + .failure(); + + // Should successfully pull with force flag + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"]) + .await + .success(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("e2e-user/{dataset_name}").as_str()) + .unwrap() + .to_string() + }; + + // 2. Push command + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Add the dataset + { + kamu_in_push_workspace + .add_dataset(DatasetSnapshot { + name: DatasetAlias::new(None, dataset_name.clone()), + kind: opendatafabric::DatasetKind::Root, + metadata: vec![ + AddPushSource { + source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), + read: ReadStepNdJson { + schema: Some(vec![ + "match_time TIMESTAMP".to_owned(), + "match_id BIGINT".to_owned(), + "player_id STRING".to_owned(), + "score BIGINT".to_owned(), + ]), + ..Default::default() + } + .into(), + preprocess: None, + merge: MergeStrategyLedger { + primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], + } + .into(), + } + .into(), + SetVocab { + event_time_column: Some("match_time".to_owned()), + ..Default::default() + } + .into(), + ], + }) + .await; + } + + // Ingest data to the dataset + { + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + } + + // Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Dataset push without storing alias + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ]) + .await + .success(); + + // Check alias should be empty + let aliases = kamu_in_push_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert!(aliases.is_empty()); + + // Dataset push with storing alias + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ]) + .await + .success(); + + let aliases = kamu_in_push_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert_eq!( + aliases, + vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Push".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }] + ) + } + + // 3. Pull command + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Dataset pull without storing alias + kamu_in_pull_workspace + .execute([ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ]) + .await + .success(); + + // Check alias should be empty + let aliases = kamu_in_pull_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert!(aliases.is_empty()); + + // Delete local dataset + kamu_in_pull_workspace + .execute(["--yes", "delete", dataset_name.as_str()]) + .await + .success(); + + // Dataset pull with storing alias + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) + .await + .success(); + + let aliases = kamu_in_pull_workspace + .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) + .await; + assert_eq!( + aliases, + vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Pull".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }] + ) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + kamu_api_server_client + .create_player_scores_dataset_with_data( + &token, + Some(&AccountName::new_unchecked("e2e-user")), + ) + .await; + + let kamu_api_server_dataset_endpoint = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("e2e-user/{dataset_name}").as_str()) + .unwrap() + .to_string() + }; + + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + let new_dataset_name = DatasetName::new_unchecked("foo"); + kamu_in_pull_workspace + .execute([ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--as", + new_dataset_name.as_str(), + ]) + .await + .success(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join("e2e-user/player-scores") + .unwrap() + .to_string() + }; + + // 2. Pushing the dataset to the API server + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2.1. Add datasets + { + kamu_in_push_workspace + .add_dataset(DatasetSnapshot { + name: DatasetAlias::new(None, dataset_name.clone()), + kind: opendatafabric::DatasetKind::Root, + metadata: vec![ + AddPushSource { + source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), + read: ReadStepNdJson { + schema: Some(vec![ + "match_time TIMESTAMP".to_owned(), + "match_id BIGINT".to_owned(), + "player_id STRING".to_owned(), + "score BIGINT".to_owned(), + ]), + ..Default::default() + } + .into(), + preprocess: None, + merge: MergeStrategyLedger { + primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], + } + .into(), + } + .into(), + SetVocab { + event_time_column: Some("match_time".to_owned()), + ..Default::default() + } + .into(), + ], + }) + .await; + + kamu_in_push_workspace + .add_dataset(DatasetSnapshot { + name: DatasetAlias::new(None, dataset_name.clone()), + kind: opendatafabric::DatasetKind::Derivative, + metadata: vec![MetadataFactory::set_transform() + .inputs_from_refs([dataset_name.clone()]) + .transform( + MetadataFactory::transform() + .engine("datafusion") + .query( + "SELECT + match_time, + match_id, + player_id, + score + FROM player-scores", + ) + .build(), + ) + .build() + .into()], + }) + .await; + } + + // 2.1. Ingest data to the dataset + { + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Push all datasets should fail + kamu_in_push_workspace + .execute(["push", "--all"]) + .await + .failure(); + } + + // 3. Pulling the dataset from the API server + // { + // let kamu_in_pull_workspace = + // KamuCliPuppet::new_workspace_tmp().await; + + // kamu_in_pull_workspace + // .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) + // .await + // .success(); + // } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 09ae75317b..71bb960b79 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -29,6 +29,8 @@ pub trait KamuCliPuppetExt { async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot); + async fn ingest_data(&self, dataset_name: &DatasetName, data: &str); + async fn get_list_of_repo_aliases(&self, dataset_ref: &DatasetRef) -> Vec; async fn complete(&self, input: T, current: usize) -> Vec @@ -176,6 +178,18 @@ impl KamuCliPuppetExt for KamuCliPuppet { kamu_data_utils::testing::assert_data_eq(df.clone(), expected_data).await; kamu_data_utils::testing::assert_schema_eq(df.schema(), expected_schema); } + + async fn ingest_data(&self, dataset_name: &DatasetName, data: &str) { + let dataset_data_path = self + .workspace_path() + .join(format!("{dataset_name}.data.ndjson")); + + std::fs::write(dataset_data_path.clone(), data).unwrap(); + + self.execute(["ingest", dataset_name, dataset_data_path.to_str().unwrap()]) + .await + .success(); + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From b0c97f136599884380fb5336582698f1df6e500c Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Thu, 10 Oct 2024 19:54:01 +0200 Subject: [PATCH 05/16] Add s3 tests --- .../tests/test_smart_transfer_protocol.rs | 45 + .../src/test_smart_transfer_protocol.rs | 913 ++++++++++++++---- 2 files changed, 775 insertions(+), 183 deletions(-) diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index 0e7313b201..a8645d0e5c 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -68,3 +68,48 @@ kamu_cli_run_api_server_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 356f133cec..a75cab1236 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -7,27 +7,19 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use core::panic; - -use kamu::testing::MetadataFactory; -use kamu_cli_e2e_common::{KamuApiServerClient, KamuApiServerClientExt}; +use std::str::FromStr; + +use chrono::DateTime; +use kamu::testing::LocalS3Server; +use kamu_cli_e2e_common::{ + KamuApiServerClient, + KamuApiServerClientExt, + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; use kamu_cli_puppet::KamuCliPuppet; -use opendatafabric::{ - AccountName, - AddPushSource, - DatasetAlias, - DatasetName, - DatasetSnapshot, - MergeStrategyLedger, - ReadStepNdJson, - SetTransform, - SetVocab, - SourceState, - Transform, - TransformInput, - TransformSql, -}; +use opendatafabric::{AccountName, DatasetName}; use reqwest::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -59,37 +51,9 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer // 2.1. Add the dataset { kamu_in_push_workspace - .add_dataset(DatasetSnapshot { - name: DatasetAlias::new(None, dataset_name.clone()), - kind: opendatafabric::DatasetKind::Root, - metadata: vec![ - AddPushSource { - source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), - read: ReadStepNdJson { - schema: Some(vec![ - "match_time TIMESTAMP".to_owned(), - "match_id BIGINT".to_owned(), - "player_id STRING".to_owned(), - "score BIGINT".to_owned(), - ]), - ..Default::default() - } - .into(), - preprocess: None, - merge: MergeStrategyLedger { - primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], - } - .into(), - } - .into(), - SetVocab { - event_time_column: Some("match_time".to_owned()), - ..Default::default() - } - .into(), - ], - }) - .await; + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); } // 2.1. Ingest data to the dataset @@ -168,37 +132,9 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli // 2.1. Add the dataset { kamu_in_push_workspace - .add_dataset(DatasetSnapshot { - name: DatasetAlias::new(None, dataset_name.clone()), - kind: opendatafabric::DatasetKind::Root, - metadata: vec![ - AddPushSource { - source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), - read: ReadStepNdJson { - schema: Some(vec![ - "match_time TIMESTAMP".to_owned(), - "match_id BIGINT".to_owned(), - "player_id STRING".to_owned(), - "score BIGINT".to_owned(), - ]), - ..Default::default() - } - .into(), - preprocess: None, - merge: MergeStrategyLedger { - primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], - } - .into(), - } - .into(), - SetVocab { - event_time_column: Some("match_time".to_owned()), - ..Default::default() - } - .into(), - ], - }) - .await; + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); } // 2.1. Ingest data to the dataset @@ -341,37 +277,9 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe // Add the dataset { kamu_in_push_workspace - .add_dataset(DatasetSnapshot { - name: DatasetAlias::new(None, dataset_name.clone()), - kind: opendatafabric::DatasetKind::Root, - metadata: vec![ - AddPushSource { - source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), - read: ReadStepNdJson { - schema: Some(vec![ - "match_time TIMESTAMP".to_owned(), - "match_id BIGINT".to_owned(), - "player_id STRING".to_owned(), - "score BIGINT".to_owned(), - ]), - ..Default::default() - } - .into(), - preprocess: None, - merge: MergeStrategyLedger { - primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], - } - .into(), - } - .into(), - SetVocab { - event_time_column: Some("match_time".to_owned()), - ..Default::default() - } - .into(), - ], - }) - .await; + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); } // Ingest data to the dataset @@ -438,7 +346,7 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe kind: "Push".to_string(), alias: kamu_api_server_dataset_endpoint.clone(), }] - ) + ); } // 3. Pull command @@ -483,7 +391,7 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe kind: "Pull".to_string(), alias: kamu_api_server_dataset_endpoint.clone(), }] - ) + ); } } @@ -534,11 +442,271 @@ pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClient) { let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { + let (kamu_api_server_root_dataset_endpoint, kamu_api_server_derivative_dataset_endpoint) = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + ( + dataset_endpoint + .clone() + .join(format!("e2e-user/{dataset_name}").as_str()) + .unwrap() + .to_string(), + dataset_endpoint + .clone() + .join(format!("e2e-user/{dataset_derivative_name}").as_str()) + .unwrap() + .to_string(), + ) + }; + + let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2. Pushing datasets to the API server + { + kamu_in_push_workspace + .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + + // 2.1. Add datasets + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu_in_push_workspace + .execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + } + + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Push all datasets should fail + kamu_in_push_workspace + .execute(["push", "--all"]) + .await + .failure(); + + // Push datasets one by one + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ]) + .await + .success(); + + kamu_in_push_workspace + .execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + kamu_in_push_workspace + .execute([ + "push", + dataset_derivative_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ]) + .await + .success(); + } + + // 3. Pulling datasets from the API server + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // Pull datasets one by one and check data + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_root_dataset_endpoint.as_str()]) + .await + .success(); + + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_derivative_dataset_endpoint.as_str()]) + .await + .success(); + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Update remote datasets + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + kamu_in_push_workspace + .execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ]) + .await + .success(); + kamu_in_push_workspace + .execute([ + "push", + dataset_derivative_name.as_str(), + "--to", + kamu_api_server_derivative_dataset_endpoint.as_str(), + ]) + .await + .success(); + + // Pull all datasets + kamu_in_pull_workspace + .execute(["pull", "--all"]) + .await + .success(); + + // Perform dataslices checks + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | Bob | 90 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 2 | Bob | 90 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_root_dataset_endpoint = { let base_url = kamu_api_server_client.get_base_url(); let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); @@ -547,75 +715,387 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien dataset_endpoint.set_port(base_url.port()).unwrap(); dataset_endpoint - .join("e2e-user/player-scores") + .clone() + .join(format!("e2e-user/{dataset_name}").as_str()) .unwrap() .to_string() }; - // 2. Pushing the dataset to the API server + let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2. Pushing datasets to the API server { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + kamu_in_push_workspace + .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); // 2.1. Add datasets { kamu_in_push_workspace - .add_dataset(DatasetSnapshot { - name: DatasetAlias::new(None, dataset_name.clone()), - kind: opendatafabric::DatasetKind::Root, - metadata: vec![ - AddPushSource { - source_name: SourceState::DEFAULT_SOURCE_NAME.to_string(), - read: ReadStepNdJson { - schema: Some(vec![ - "match_time TIMESTAMP".to_owned(), - "match_id BIGINT".to_owned(), - "player_id STRING".to_owned(), - "score BIGINT".to_owned(), - ]), - ..Default::default() - } - .into(), - preprocess: None, - merge: MergeStrategyLedger { - primary_key: vec!["match_id".to_owned(), "player_id".to_owned()], - } - .into(), - } - .into(), - SetVocab { - event_time_column: Some("match_time".to_owned()), - ..Default::default() - } - .into(), - ], - }) - .await; + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + } + + // 2.1. Ingest data to the dataset + { + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); kamu_in_push_workspace - .add_dataset(DatasetSnapshot { - name: DatasetAlias::new(None, dataset_name.clone()), - kind: opendatafabric::DatasetKind::Derivative, - metadata: vec![MetadataFactory::set_transform() - .inputs_from_refs([dataset_name.clone()]) - .transform( - MetadataFactory::transform() - .engine("datafusion") - .query( - "SELECT - match_time, - match_id, - player_id, - score - FROM player-scores", - ) - .build(), - ) - .build() - .into()], - }) + .ingest_data(&dataset_name, data) .await; } + // 2.2. Login to the API server + kamu_in_push_workspace + .execute([ + "login", + kamu_api_server_client.get_base_url().as_str(), + "--access-token", + token.as_str(), + ]) + .await + .success(); + + // Push all datasets should fail + kamu_in_push_workspace + .execute(["push", dataset_name.as_str(), "--recursive"]) + .await + .failure(); + + // Push dataset + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ]) + .await + .success(); + } + + // 3. Pulling datasets from the API server + { + let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + kamu_in_pull_workspace + .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + + // Pull datasets one by one and check data + kamu_in_pull_workspace + .execute(["pull", kamu_api_server_root_dataset_endpoint.as_str()]) + .await + .success(); + + kamu_in_pull_workspace + .execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + kamu_in_pull_workspace + .execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Update remote datasets + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu_in_push_workspace + .ingest_data(&dataset_name, data) + .await; + kamu_in_push_workspace + .execute([ + "push", + dataset_name.as_str(), + "--to", + kamu_api_server_root_dataset_endpoint.as_str(), + ]) + .await + .success(); + + // Pull all datasets + kamu_in_pull_workspace + .execute(["pull", dataset_derivative_name.as_str(), "--recursive"]) + .await + .success(); + + // Perform dataslices checks + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | Bob | 90 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 2 | Bob | 90 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + + kamu_in_pull_workspace + .assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute([ + "pull", + dataset_name.as_str(), + "--set-watermark", + "2000-01-01T00:00:00Z", + ]) + .await + .success(); + + // let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + // assert!( + // stdout.contains(indoc::indoc!(r#"Committed new block"#).trim()), + // "Unexpected output:\n{stdout}", + // ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { + kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + kamu.execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; + + // Compact root dataset + kamu.execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + // Pull derivative should fail + kamu.execute(["pull", dataset_derivative_name.as_str()]) + .await + .failure(); + + // Add new data to root dataset + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + kamu.execute([ + "pull", + dataset_derivative_name.as_str(), + "--reset-derivatives-on-diverged-input", + ]) + .await + .success(); + + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerClient) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + // 1. Grub a token + let token = kamu_api_server_client.login_as_e2e_user().await; + + let kamu_api_server_dataset_endpoint = { + let base_url = kamu_api_server_client.get_base_url(); + + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("e2e-user/{dataset_name}").as_str()) + .unwrap() + .to_string() + }; + + // 2. Pushing the dataset to the API server + { + let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + + // 2.1. Add the dataset + { + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + } + // 2.1. Ingest data to the dataset { let data = indoc::indoc!( @@ -641,23 +1121,90 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien .await .success(); - // Push all datasets should fail kamu_in_push_workspace - .execute(["push", "--all"]) + .execute([ + "push", + "player-scores", + "--to", + kamu_api_server_dataset_endpoint.as_str(), + "--visibility", + "private", + ]) .await - .failure(); + .success(); + + // ToDo add visibility check } +} - // 3. Pulling the dataset from the API server - // { - // let kamu_in_pull_workspace = - // KamuCliPuppet::new_workspace_tmp().await; - - // kamu_in_pull_workspace - // .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) - // .await - // .success(); - // } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { + kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let s3_server = LocalS3Server::new().await; + + let dataset_url = format!("{}/e2e-user/{dataset_name}", s3_server.url); + // Push dataset + kamu.execute(["push", dataset_name.as_str(), "--to", dataset_url.as_str()]) + .await + .success(); + + { + let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + + kamu_in_pull_workspace + .execute(["pull", dataset_url.as_str()]) + .await + .success(); + + let expected_schema = indoc::indoc!( + r#" + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + let expected_data = indoc::indoc!( + r#" + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | + +--------+----+----------------------+----------------------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice(&dataset_name, expected_schema, expected_data) + .await; + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From ff8281d11e90f2c67fef8c4ab220ffaaa3ada501 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Thu, 10 Oct 2024 20:27:07 +0200 Subject: [PATCH 06/16] Remove unused code --- .../app/cli/repo-tests/src/test_smart_transfer_protocol.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index a75cab1236..0500eee174 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -1147,13 +1147,6 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { .await .success(); - kamu.execute_with_input( - ["add", "--stdin"], - DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, - ) - .await - .success(); - let data = indoc::indoc!( r#" {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} From 27838efff2231b5bd6fc5a926234363ddc5061b2 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Thu, 10 Oct 2024 22:03:20 +0200 Subject: [PATCH 07/16] Test transform engine error --- .../common/src/kamu_api_server_client_ext.rs | 16 +--- .../src/test_smart_transfer_protocol.rs | 80 +++++++++---------- 2 files changed, 43 insertions(+), 53 deletions(-) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index c08e2d964a..4347f84b22 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -57,30 +57,20 @@ pub const DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR: &str = indoc::indoc!( alias: player_scores transform: kind: Sql - engine: risingwave + engine: flink queries: - - alias: leaderboard - # Note we are using explicit `crate materialized view` statement below - # because RW does not currently support Top-N queries directly on sinks. - # - # Note `partition by 1` is currently required by RW engine - # See: https://docs.risingwave.com/docs/current/window-functions/#syntax - query: | - create materialized view leaderboard as + - query: | select * from ( select - row_number() over (partition by 1 order by score desc) as place, match_time, match_id, player_id, score from player_scores ) - where place <= 2 - - query: | - select * from leaderboard + where score > 90 - kind: SetVocab eventTimeColumn: match_time "# diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 0500eee174..269f920fff 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -616,13 +616,13 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - kamu_in_pull_workspace - .assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu_in_pull_workspace + // .assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; // Update remote datasets let data = indoc::indoc!( @@ -687,13 +687,13 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - kamu_in_pull_workspace - .assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu_in_pull_workspace + // .assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; } } @@ -855,13 +855,13 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - kamu_in_pull_workspace - .assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu_in_pull_workspace + // .assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; // Update remote datasets let data = indoc::indoc!( @@ -913,13 +913,13 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - kamu_in_pull_workspace - .assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu_in_pull_workspace + // .assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; } } @@ -1004,12 +1004,12 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); - kamu.assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu.assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; // Compact root dataset kamu.execute([ @@ -1054,12 +1054,12 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); - kamu.assert_last_data_slice( - &dataset_derivative_name, - expected_derivative_schema, - expected_derivative_data, - ) - .await; + // kamu.assert_last_data_slice( + // &dataset_derivative_name, + // expected_derivative_schema, + // expected_derivative_data, + // ) + // .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From f48972aa2d2293f55a230673f218c160456dfc78 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Fri, 11 Oct 2024 10:45:09 +0200 Subject: [PATCH 08/16] Revert check --- .../common/src/kamu_api_server_client_ext.rs | 16 +++- .../tests/test_smart_transfer_protocol.rs | 6 +- .../src/test_smart_transfer_protocol.rs | 80 +++++++++---------- 3 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index 4347f84b22..c08e2d964a 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -57,20 +57,30 @@ pub const DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR: &str = indoc::indoc!( alias: player_scores transform: kind: Sql - engine: flink + engine: risingwave queries: - - query: | + - alias: leaderboard + # Note we are using explicit `crate materialized view` statement below + # because RW does not currently support Top-N queries directly on sinks. + # + # Note `partition by 1` is currently required by RW engine + # See: https://docs.risingwave.com/docs/current/window-functions/#syntax + query: | + create materialized view leaderboard as select * from ( select + row_number() over (partition by 1 order by score desc) as place, match_time, match_id, player_id, score from player_scores ) - where score > 90 + where place <= 2 + - query: | + select * from leaderboard - kind: SetVocab eventTimeColumn: match_time "# diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index a8645d0e5c..a6722dcde1 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -64,7 +64,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "engine, ingest, datafusion" + extra_test_groups = "engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -75,7 +75,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "engine, ingest, datafusion" + extra_test_groups = "engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -90,7 +90,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, - extra_test_groups = "engine, ingest, datafusion" + extra_test_groups = "engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 269f920fff..0500eee174 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -616,13 +616,13 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - // kamu_in_pull_workspace - // .assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; // Update remote datasets let data = indoc::indoc!( @@ -687,13 +687,13 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - // kamu_in_pull_workspace - // .assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; } } @@ -855,13 +855,13 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - // kamu_in_pull_workspace - // .assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; // Update remote datasets let data = indoc::indoc!( @@ -913,13 +913,13 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe kamu_in_pull_workspace .assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; - // kamu_in_pull_workspace - // .assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu_in_pull_workspace + .assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; } } @@ -1004,12 +1004,12 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); - // kamu.assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; // Compact root dataset kamu.execute([ @@ -1054,12 +1054,12 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ "# ); - // kamu.assert_last_data_slice( - // &dataset_derivative_name, - // expected_derivative_schema, - // expected_derivative_data, - // ) - // .await; + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From f6bf6d8c3c6261da14d5d8b0d181213e44c69c90 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Fri, 11 Oct 2024 11:04:47 +0200 Subject: [PATCH 09/16] Add container extra groups --- .../cli/inmem/tests/tests/test_smart_transfer_protocol.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index a6722dcde1..9b180786b9 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -64,7 +64,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -75,7 +75,7 @@ kamu_cli_run_api_server_e2e_test!( options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), - extra_test_groups = "engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -90,7 +90,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, - extra_test_groups = "engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -109,7 +109,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, - extra_test_groups = "engine, ingest, datafusion" + extra_test_groups = "containerized, engine, ingest, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 9ff33cdeb5fe039af69365e541da35084a054510 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Fri, 11 Oct 2024 16:23:50 +0200 Subject: [PATCH 10/16] Add verify and compact commands --- src/app/cli/src/commands/verify_command.rs | 1 - .../app/cli/inmem/tests/tests/commands/mod.rs | 2 + .../tests/commands/test_compact_command.rs | 36 +++ .../tests/commands/test_verify_command.rs | 36 +++ .../app/cli/repo-tests/src/commands/mod.rs | 4 + .../src/commands/test_compact_command.rs | 208 ++++++++++++++++++ .../src/commands/test_verify_command.rs | 150 +++++++++++++ src/e2e/app/cli/repo-tests/src/lib.rs | 2 + .../src/test_smart_transfer_protocol.rs | 27 +-- .../src/kamu_cli_puppet_ext.rs | 63 +++++- 10 files changed, 512 insertions(+), 17 deletions(-) create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs create mode 100644 src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs diff --git a/src/app/cli/src/commands/verify_command.rs b/src/app/cli/src/commands/verify_command.rs index 433dcc395a..8aab660b28 100644 --- a/src/app/cli/src/commands/verify_command.rs +++ b/src/app/cli/src/commands/verify_command.rs @@ -152,7 +152,6 @@ impl VerifyCommand { Ok(self .verification_svc - .clone() .verify_multi(filtered_requests, options, listener) .await) } diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs index 695c4f9d47..b25d7e1c40 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/mod.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. mod test_add_command; +mod test_compact_command; mod test_complete_command; mod test_config_command; mod test_delete_command; @@ -18,3 +19,4 @@ mod test_repo_alias_command; mod test_sql_command; mod test_system_api_server_gql_query; mod test_system_generate_token_command; +mod test_verify_command; diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs new file mode 100644 index 0000000000..6ff90ee7b5 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_hard + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_compact_verify + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs new file mode 100644 index 0000000000..0a4b5598a2 --- /dev/null +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs @@ -0,0 +1,36 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::prelude::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_recursive + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_verify_integrity + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/mod.rs b/src/e2e/app/cli/repo-tests/src/commands/mod.rs index d3763d8426..6745404916 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/mod.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/mod.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. mod test_add_command; +mod test_compact_command; mod test_complete_command; mod test_config_command; mod test_delete_command; @@ -18,8 +19,10 @@ mod test_repo_alias_command; mod test_sql_command; mod test_system_api_server_gql_query; mod test_system_generate_token_command; +mod test_verify_command; pub use test_add_command::*; +pub use test_compact_command::*; pub use test_complete_command::*; pub use test_config_command::*; pub use test_delete_command::*; @@ -30,3 +33,4 @@ pub use test_repo_alias_command::*; pub use test_sql_command::*; pub use test_system_api_server_gql_query::*; pub use test_system_generate_token_command::*; +pub use test_verify_command::*; diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs new file mode 100644 index 0000000000..d665c14fdb --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs @@ -0,0 +1,208 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; + +use kamu_cli_e2e_common::DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::{DatasetName, MetadataEvent}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_hard(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 1, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::AddData(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_keep_metadata_only(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 2, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::SetDataSchema(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_compact_verify(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; + + let assert = kamu + .execute([ + "--yes", + "system", + "compact", + dataset_name.as_str(), + "--hard", + "--verify", + ]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains( + indoc::indoc!( + r#" + verify with dataset_ref: player-scores + "# + ) + .trim() + ), + "Unexpected output:\n{stderr}", + ); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) were compacted + "# + )), + "Unexpected output:\n{stderr}", + ); + + let blocks_after_compacting = kamu.list_blocks(&dataset_name).await; + assert_eq!( + blocks_before_compacting.len() - 1, + blocks_after_compacting.len() + ); + assert_matches!( + blocks_after_compacting.first().unwrap().block.event, + MetadataEvent::AddData(_) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs new file mode 100644 index 0000000000..088c6e1b27 --- /dev/null +++ b/src/e2e/app/cli/repo-tests/src/commands/test_verify_command.rs @@ -0,0 +1,150 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_cli_e2e_common::{ + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; +use kamu_cli_puppet::extensions::KamuCliPuppetExt; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric::DatasetName; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_regular_dataset(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let assert = kamu + .execute(["verify", dataset_name.as_str()]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_recursive(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + kamu.execute(["pull", dataset_derivative_name.as_str()]) + .await + .success(); + + // Call verify without recursive flag + let assert = kamu + .execute(["verify", dataset_derivative_name.as_str()]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); + + // Call verify wit recursive flag + let assert = kamu + .execute(["verify", dataset_derivative_name.as_str(), "--recursive"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 2 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_verify_integrity(kamu: KamuCliPuppet) { + let dataset_name = DatasetName::new_unchecked("player-scores"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + let data = indoc::indoc!( + r#" + {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} + "#, + ); + + kamu.ingest_data(&dataset_name, data).await; + + let assert = kamu + .execute(["verify", dataset_name.as_str(), "--integrity"]) + .await + .success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(indoc::indoc!( + r#" + 1 dataset(s) are valid + "# + )), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/lib.rs b/src/e2e/app/cli/repo-tests/src/lib.rs index 2bd7d97f92..51d5ab6b2f 100644 --- a/src/e2e/app/cli/repo-tests/src/lib.rs +++ b/src/e2e/app/cli/repo-tests/src/lib.rs @@ -7,6 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +#![feature(assert_matches)] + mod commands; mod test_auth; mod test_flow; diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 0500eee174..a6ca2dbdc2 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -932,21 +932,22 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { .await .success(); - kamu.execute([ - "pull", - dataset_name.as_str(), - "--set-watermark", - "2000-01-01T00:00:00Z", - ]) - .await - .success(); + let assert = kamu + .execute([ + "pull", + dataset_name.as_str(), + "--set-watermark", + "2000-01-01T00:00:00Z", + ]) + .await + .success(); - // let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); - // assert!( - // stdout.contains(indoc::indoc!(r#"Committed new block"#).trim()), - // "Unexpected output:\n{stdout}", - // ); + assert!( + stderr.contains(indoc::indoc!(r#"Committed new block"#).trim()), + "Unexpected output:\n{stderr}", + ); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 71bb960b79..40a23d74d9 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -14,9 +14,21 @@ use std::path::PathBuf; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion::prelude::{ParquetReadOptions, SessionContext}; -use opendatafabric::serde::yaml::{DatasetKindDef, YamlDatasetSnapshotSerializer}; -use opendatafabric::serde::DatasetSnapshotSerializer; -use opendatafabric::{DatasetID, DatasetKind, DatasetName, DatasetRef, DatasetSnapshot, Multihash}; +use opendatafabric::serde::yaml::{ + DatasetKindDef, + YamlDatasetSnapshotSerializer, + YamlMetadataBlockDeserializer, +}; +use opendatafabric::serde::{DatasetSnapshotSerializer, MetadataBlockDeserializer}; +use opendatafabric::{ + DatasetID, + DatasetKind, + DatasetName, + DatasetRef, + DatasetSnapshot, + MetadataBlock, + Multihash, +}; use serde::Deserialize; use crate::KamuCliPuppet; @@ -29,6 +41,8 @@ pub trait KamuCliPuppetExt { async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot); + async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; + async fn ingest_data(&self, dataset_name: &DatasetName, data: &str); async fn get_list_of_repo_aliases(&self, dataset_ref: &DatasetRef) -> Vec; @@ -190,6 +204,43 @@ impl KamuCliPuppetExt for KamuCliPuppet { .await .success(); } + + async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec { + let assert = self + .execute(["log", dataset_name.as_str(), "--output-format", "yaml"]) + .await + .success(); + + let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + + // TODO: Don't parse the output, after implementation: + // `kamu log`: support `--output-format json` + // https://github.com/kamu-data/kamu-cli/issues/887 + + stdout + .split("---") + .skip(1) + .map(str::trim) + .map(|block_data| { + let Some(pos) = block_data.find('\n') else { + unreachable!() + }; + let (first_line_with_block_hash, metadata_block_str) = block_data.split_at(pos); + + let block_hash = first_line_with_block_hash + .strip_prefix("# Block: ") + .unwrap(); + let block = YamlMetadataBlockDeserializer {} + .read_manifest(metadata_block_str.as_ref()) + .unwrap(); + + BlockRecord { + block_hash: Multihash::from_multibase(block_hash).unwrap(), + block, + } + }) + .collect() + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -225,4 +276,10 @@ pub struct RepoAlias { pub alias: String, } +#[derive(Debug, PartialEq, Eq)] +pub struct BlockRecord { + pub block_hash: Multihash, + pub block: MetadataBlock, +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 1130fa5bc1f8816d0b2968b1fdb96e135dea55bf Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Fri, 11 Oct 2024 18:31:56 +0200 Subject: [PATCH 11/16] Fix grammar --- src/app/cli/src/commands/verify_command.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/cli/src/commands/verify_command.rs b/src/app/cli/src/commands/verify_command.rs index 8aab660b28..ac3d257789 100644 --- a/src/app/cli/src/commands/verify_command.rs +++ b/src/app/cli/src/commands/verify_command.rs @@ -192,14 +192,14 @@ impl VerifyCommand { let mut current_missed_dependencies = vec![]; - for dependecy in summary.dependencies { + for dependency in summary.dependencies { if self .dataset_repo - .resolve_dataset_ref(&DatasetRef::ID(dependecy.clone())) + .resolve_dataset_ref(&DatasetRef::ID(dependency.clone())) .await .is_err() { - current_missed_dependencies.push(dependecy.to_string()); + current_missed_dependencies.push(dependency.to_string()); } } if !current_missed_dependencies.is_empty() { From cb736053c8ee0e9a81438304dbfced15eef0b206 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 13:11:24 +0200 Subject: [PATCH 12/16] Fix review commets --- .../common/src/kamu_api_server_client_ext.rs | 22 +- .../tests/test_smart_transfer_protocol.rs | 8 + .../src/commands/test_compact_command.rs | 78 +- .../src/test_smart_transfer_protocol.rs | 1007 +++++++++-------- .../src/kamu_cli_puppet_ext.rs | 50 +- 5 files changed, 622 insertions(+), 543 deletions(-) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index a8a53ac4e8..bee28b2b52 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use lazy_static::lazy_static; -use opendatafabric::{DatasetAlias, DatasetName}; +use opendatafabric::{AccountName, DatasetAlias, DatasetName}; use reqwest::{Method, StatusCode}; use crate::{KamuApiServerClient, RequestBody}; @@ -134,6 +134,8 @@ pub const DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_3: &str = indoc::i "# ); +pub const E2E_USER_ACCOUNT_NAME_STR: &str = "e2e-user"; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub type AccessToken = String; @@ -153,8 +155,11 @@ pub trait KamuApiServerClientExt { async fn create_player_scores_dataset(&self, token: &AccessToken) -> DatasetId; - /// NOTE: only for single-tenant workspaces - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId; + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option, + ) -> DatasetId; async fn create_leaderboard(&self, token: &AccessToken) -> DatasetId; @@ -249,14 +254,21 @@ impl KamuApiServerClientExt for KamuApiServerClient { .await } - async fn create_player_scores_dataset_with_data(&self, token: &AccessToken) -> DatasetId { + async fn create_player_scores_dataset_with_data( + &self, + token: &AccessToken, + account_name_maybe: Option, + ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; // TODO: Use the alias from the reply, after fixing the bug: // https://github.com/kamu-data/kamu-cli/issues/891 // At the moment, only single-tenant - let dataset_alias = DatasetAlias::new(None, DatasetName::new_unchecked("player-scores")); + let dataset_alias = DatasetAlias::new( + account_name_maybe, + DatasetName::new_unchecked("player-scores"), + ); self.ingest_data( &dataset_alias, diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index 9b180786b9..c99f906c05 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -113,3 +113,11 @@ kamu_cli_execute_command_e2e_test!( ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs index d665c14fdb..316359eb49 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs @@ -9,7 +9,11 @@ use std::assert_matches::assert_matches; -use kamu_cli_e2e_common::DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR; +use kamu_cli_e2e_common::{ + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, +}; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use opendatafabric::{DatasetName, MetadataEvent}; @@ -23,20 +27,16 @@ pub async fn test_compact_hard(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; @@ -82,20 +82,16 @@ pub async fn test_compact_keep_metadata_only(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; @@ -142,20 +138,16 @@ pub async fn test_compact_verify(kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; let blocks_before_compacting = kamu.list_blocks(&dataset_name).await; diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index a6ca2dbdc2..3d7b5a6b34 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -15,7 +15,10 @@ use kamu_cli_e2e_common::{ KamuApiServerClient, KamuApiServerClientExt, DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, + E2E_USER_ACCOUNT_NAME_STR, }; use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; use kamu_cli_puppet::KamuCliPuppet; @@ -30,19 +33,11 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join("e2e-user/player-scores") - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); // 2. Pushing the dataset to the API server { @@ -58,15 +53,11 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer // 2.1. Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -82,25 +73,29 @@ pub async fn test_smart_push_pull_sequence(kamu_api_server_client: KamuApiServer .success(); // 2.3. Push the dataset to the API server - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", - "player-scores", + dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; } // 3. Pulling the dataset from the API server { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; } } @@ -111,19 +106,11 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); // 2. Pushing the dataset to the API server { @@ -139,15 +126,11 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli // 2.1. Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -163,15 +146,17 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli .success(); // Initial dataset push - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; // Hard compact dataset kamu_in_push_workspace @@ -187,27 +172,31 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli .success(); // Should fail without force flag - kamu_in_push_workspace - .execute([ + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), - ]) - .await - .failure(); + ], + "Failed to push 1 dataset(s)", + ) + .await; // Should successfully push with force flag - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), "--force", - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; } // 3. Pulling the dataset from the API server @@ -215,37 +204,41 @@ pub async fn test_smart_force_push_pull(kamu_api_server_client: KamuApiServerCli let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Call with no-alias flag to avoid remote ingest checking in next step - kamu_in_pull_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ "pull", kamu_api_server_dataset_endpoint.as_str(), "--no-alias", - ]) - .await - .success(); + ], + "1 dataset(s) updated", + ) + .await; // Ingest data in pulled dataset - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Chuck", "score": 90} - "#, - ); kamu_in_pull_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) .await; // Should fail without force flag - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) - .await - .failure(); + run_and_assert_command_failure( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "Failed to update 1 dataset(s)", + ) + .await; // Should successfully pull with force flag - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"], + "1 dataset(s) updated", + ) + .await; } } @@ -256,19 +249,11 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); // 2. Push command { @@ -284,15 +269,11 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe // Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -308,16 +289,18 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe .success(); // Dataset push without storing alias - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), "--no-alias", - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; // Check alias should be empty let aliases = kamu_in_push_workspace @@ -326,27 +309,27 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe assert!(aliases.is_empty()); // Dataset push with storing alias - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) up-to-date", + ) + .await; let aliases = kamu_in_push_workspace .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) .await; - assert_eq!( - aliases, - vec![RepoAlias { - dataset: dataset_name.clone(), - kind: "Push".to_string(), - alias: kamu_api_server_dataset_endpoint.clone(), - }] - ); + let expected_aliases = vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Push".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }]; + pretty_assertions::assert_eq!(aliases, expected_aliases); } // 3. Pull command @@ -354,14 +337,16 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Dataset pull without storing alias - kamu_in_pull_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ "pull", kamu_api_server_dataset_endpoint.as_str(), "--no-alias", - ]) - .await - .success(); + ], + "1 dataset(s) updated", + ) + .await; // Check alias should be empty let aliases = kamu_in_pull_workspace @@ -376,22 +361,22 @@ pub async fn test_smart_push_pull_add_alias(kamu_api_server_client: KamuApiServe .success(); // Dataset pull with storing alias - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_dataset_endpoint.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; let aliases = kamu_in_pull_workspace .get_list_of_repo_aliases(&opendatafabric::DatasetRef::from(dataset_name.clone())) .await; - assert_eq!( - aliases, - vec![RepoAlias { - dataset: dataset_name.clone(), - kind: "Pull".to_string(), - alias: kamu_api_server_dataset_endpoint.clone(), - }] - ); + let expected_aliases = vec![RepoAlias { + dataset: dataset_name.clone(), + kind: "Pull".to_string(), + alias: kamu_api_server_dataset_endpoint.clone(), + }]; + pretty_assertions::assert_eq!(aliases, expected_aliases); } } @@ -401,40 +386,44 @@ pub async fn test_smart_pull_as(kamu_api_server_client: KamuApiServerClient) { let dataset_name = DatasetName::new_unchecked("player-scores"); // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; + kamu_api_server_client .create_player_scores_dataset_with_data( &token, - Some(&AccountName::new_unchecked("e2e-user")), + Some(AccountName::new_unchecked(E2E_USER_ACCOUNT_NAME_STR)), ) .await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; let new_dataset_name = DatasetName::new_unchecked("foo"); - kamu_in_pull_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec![ "pull", kamu_api_server_dataset_endpoint.as_str(), "--as", new_dataset_name.as_str(), - ]) + ], + "1 dataset(s) updated", + ) + .await; + + let expected_dataset_list = kamu_in_pull_workspace + .list_datasets() .await - .success(); + .into_iter() + .map(|dataset| dataset.name) + .collect::>(); + + pretty_assertions::assert_eq!(vec![new_dataset_name], expected_dataset_list); } } @@ -447,27 +436,16 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let (kamu_api_server_root_dataset_endpoint, kamu_api_server_derivative_dataset_endpoint) = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - ( - dataset_endpoint - .clone() - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string(), - dataset_endpoint - .clone() - .join(format!("e2e-user/{dataset_derivative_name}").as_str()) - .unwrap() - .to_string(), - ) - }; + let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); + let kamu_api_server_derivative_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_derivative_name, + E2E_USER_ACCOUNT_NAME_STR, + ); let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; @@ -494,15 +472,11 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien // 2.1. Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -518,36 +492,42 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien .success(); // Push all datasets should fail - kamu_in_push_workspace - .execute(["push", "--all"]) - .await - .failure(); + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec!["push", "--all"], + "Pushing all datasets is not yet supported", + ) + .await; // Push datasets one by one - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_root_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; kamu_in_push_workspace .execute(["pull", dataset_derivative_name.as_str()]) .await .success(); - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_derivative_name.as_str(), "--to", kamu_api_server_derivative_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; } // 3. Pulling datasets from the API server @@ -555,15 +535,18 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; // Pull datasets one by one and check data - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_root_dataset_endpoint.as_str()]) - .await - .success(); - - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_derivative_dataset_endpoint.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_derivative_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; let expected_schema = indoc::indoc!( r#" @@ -580,12 +563,12 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); let expected_derivative_schema = indoc::indoc!( @@ -604,12 +587,12 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -625,62 +608,70 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien .await; // Update remote datasets - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) .await; - kamu_in_push_workspace - .execute(["pull", dataset_derivative_name.as_str()]) - .await - .success(); - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_root_dataset_endpoint.as_str(), - ]) - .await - .success(); - kamu_in_push_workspace - .execute([ + ], + "1 dataset(s) pushed", + ) + .await; + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_derivative_name.as_str(), "--to", kamu_api_server_derivative_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; // Pull all datasets - kamu_in_pull_workspace - .execute(["pull", "--all"]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", "--all"], + "2 dataset(s) updated", + ) + .await; // Perform dataslices checks let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | Bob | 90 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 2 | Bob | 90 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -706,21 +697,11 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_root_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .clone() - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string() - }; - + let kamu_api_server_root_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; // 2. Pushing datasets to the API server @@ -738,15 +719,11 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe // 2.1. Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -762,21 +739,25 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .success(); // Push all datasets should fail - kamu_in_push_workspace - .execute(["push", dataset_name.as_str(), "--recursive"]) - .await - .failure(); + run_and_assert_command_failure( + &kamu_in_push_workspace, + vec!["push", dataset_name.as_str(), "--recursive"], + "Recursive push is not yet supported", + ) + .await; // Push dataset - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_root_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; } // 3. Pulling datasets from the API server @@ -786,10 +767,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); // Pull datasets one by one and check data - kamu_in_pull_workspace - .execute(["pull", kamu_api_server_root_dataset_endpoint.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", kamu_api_server_root_dataset_endpoint.as_str()], + "1 dataset(s) updated", + ) + .await; kamu_in_pull_workspace .execute_with_input( @@ -799,10 +782,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .await .success(); - kamu_in_pull_workspace - .execute(["pull", dataset_derivative_name.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; let expected_schema = indoc::indoc!( r#" @@ -819,12 +804,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); let expected_derivative_schema = indoc::indoc!( @@ -843,12 +828,12 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -864,49 +849,52 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe .await; // Update remote datasets - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90} - "#, - ); kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) .await; - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", dataset_name.as_str(), "--to", kamu_api_server_root_dataset_endpoint.as_str(), - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; // Pull all datasets - kamu_in_pull_workspace - .execute(["pull", dataset_derivative_name.as_str(), "--recursive"]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_derivative_name.as_str(), "--recursive"], + "2 dataset(s) updated", + ) + .await; // Perform dataslices checks let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | Bob | 90 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 2 | Bob | 90 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -968,42 +956,42 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; - kamu.execute(["pull", dataset_derivative_name.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; let expected_derivative_schema = indoc::indoc!( r#" - message arrow_schema { - OPTIONAL INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 place; - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# ); let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - "# + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# ); kamu.assert_last_data_slice( &dataset_derivative_name, @@ -1025,35 +1013,40 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { .success(); // Pull derivative should fail - kamu.execute(["pull", dataset_derivative_name.as_str()]) - .await - .failure(); + run_and_assert_command_failure( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "Failed to update 1 dataset(s)", + ) + .await; // Add new data to root dataset - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; - kamu.execute([ - "pull", - dataset_derivative_name.as_str(), - "--reset-derivatives-on-diverged-input", - ]) - .await - .success(); + run_and_assert_command_success( + &kamu, + vec![ + "pull", + dataset_derivative_name.as_str(), + "--reset-derivatives-on-diverged-input", + ], + "1 dataset(s) updated", + ) + .await; let expected_derivative_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - "# + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# ); kamu.assert_last_data_slice( &dataset_derivative_name, @@ -1071,19 +1064,11 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli // 1. Grub a token let token = kamu_api_server_client.login_as_e2e_user().await; - let kamu_api_server_dataset_endpoint = { - let base_url = kamu_api_server_client.get_base_url(); - - let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); - - dataset_endpoint.set_host(base_url.host_str()).unwrap(); - dataset_endpoint.set_port(base_url.port()).unwrap(); - - dataset_endpoint - .join(format!("e2e-user/{dataset_name}").as_str()) - .unwrap() - .to_string() - }; + let kamu_api_server_dataset_endpoint = get_dataset_endpoint( + kamu_api_server_client.get_base_url(), + &dataset_name, + E2E_USER_ACCOUNT_NAME_STR, + ); // 2. Pushing the dataset to the API server { @@ -1099,15 +1084,11 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli // 2.1. Ingest data to the dataset { - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - kamu_in_push_workspace - .ingest_data(&dataset_name, data) + .ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) .await; } @@ -1122,17 +1103,19 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli .await .success(); - kamu_in_push_workspace - .execute([ + run_and_assert_command_success( + &kamu_in_push_workspace, + vec![ "push", - "player-scores", + dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), "--visibility", "private", - ]) - .await - .success(); + ], + "1 dataset(s) pushed", + ) + .await; // ToDo add visibility check } @@ -1148,53 +1131,55 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { .await .success(); - let data = indoc::indoc!( - r#" - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 100} - {"match_time": "2000-01-01", "match_id": 1, "player_id": "Bob", "score": 80} - "#, - ); - - kamu.ingest_data(&dataset_name, data).await; + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; let s3_server = LocalS3Server::new().await; let dataset_url = format!("{}/e2e-user/{dataset_name}", s3_server.url); // Push dataset - kamu.execute(["push", dataset_name.as_str(), "--to", dataset_url.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu, + vec!["push", dataset_name.as_str(), "--to", dataset_url.as_str()], + "1 dataset(s) pushed", + ) + .await; { let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - kamu_in_pull_workspace - .execute(["pull", dataset_url.as_str()]) - .await - .success(); + run_and_assert_command_success( + &kamu_in_pull_workspace, + vec!["pull", dataset_url.as_str()], + "1 dataset(s) updated", + ) + .await; let expected_schema = indoc::indoc!( r#" - message arrow_schema { - REQUIRED INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# + message arrow_schema { + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# ); let expected_data = indoc::indoc!( r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - "# + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | offset | op | system_time | match_time | match_id | player_id | score | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+----------+-----------+-------+ + "# ); kamu.assert_last_data_slice(&dataset_name, expected_schema, expected_data) .await; @@ -1202,3 +1187,129 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) { + kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + let dataset_name = DatasetName::new_unchecked("player-scores"); + let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); + + kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); + + kamu.execute_with_input( + ["add", "--stdin"], + DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, + ) + .await + .success(); + + kamu.ingest_data( + &dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; + + run_and_assert_command_failure( + &kamu, + vec![ + "tail", + dataset_derivative_name.as_str(), + "--output-format", + "table", + ], + "Error: Dataset schema is not yet available: leaderboard", + ) + .await; + + run_and_assert_command_success( + &kamu, + vec!["pull", dataset_derivative_name.as_str()], + "1 dataset(s) updated", + ) + .await; + + let expected_derivative_schema = indoc::indoc!( + r#" + message arrow_schema { + OPTIONAL INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 place; + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; + } + "# + ); + + let expected_derivative_data = indoc::indoc!( + r#" + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | offset | op | system_time | match_time | place | match_id | player_id | score | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ + "# + ); + kamu.assert_last_data_slice( + &dataset_derivative_name, + expected_derivative_schema, + expected_derivative_data, + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn get_dataset_endpoint( + base_url: &Url, + dataset_name: &DatasetName, + account_name_str: &str, +) -> String { + let mut dataset_endpoint = Url::parse("odf+http://host").unwrap(); + + dataset_endpoint.set_host(base_url.host_str()).unwrap(); + dataset_endpoint.set_port(base_url.port()).unwrap(); + + dataset_endpoint + .join(format!("{account_name_str}/{dataset_name}").as_str()) + .unwrap() + .to_string() +} + +async fn run_and_assert_command_success( + kamu: &KamuCliPuppet, + args: Vec<&str>, + expected_message: &str, +) { + let assert = kamu.execute(args).await.success(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(expected_message), + "Unexpected output:\n{stderr}", + ); +} + +async fn run_and_assert_command_failure( + kamu: &KamuCliPuppet, + args: Vec<&str>, + expected_message: &str, +) { + let assert = kamu.execute(args).await.failure(); + + let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap(); + + assert!( + stderr.contains(expected_message), + "Unexpected output:\n{stderr}", + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index beadab2235..0abaddb68e 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -14,15 +14,10 @@ use std::path::PathBuf; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion::prelude::{ParquetReadOptions, SessionContext}; -use opendatafabric::serde::yaml::{ - DatasetKindDef, - YamlDatasetSnapshotSerializer, - YamlMetadataBlockDeserializer, -}; +use opendatafabric::serde::yaml::{YamlDatasetSnapshotSerializer, YamlMetadataBlockDeserializer}; use opendatafabric::serde::{DatasetSnapshotSerializer, MetadataBlockDeserializer}; use opendatafabric::{ DatasetID, - DatasetKind, DatasetName, DatasetRef, DatasetSnapshot, @@ -51,8 +46,6 @@ pub trait KamuCliPuppetExt { where T: Into + Send; - async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; - async fn start_api_server(self, e2e_data_file_path: PathBuf) -> ServerOutput; async fn assert_last_data_slice( @@ -74,6 +67,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { .success(); let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + println!("@@@@@ stdout: {:?}", stdout); serde_json::from_str(stdout).unwrap() } @@ -243,43 +237,6 @@ impl KamuCliPuppetExt for KamuCliPuppet { .await .success(); } - - async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec { - let assert = self - .execute(["log", dataset_name.as_str(), "--output-format", "yaml"]) - .await - .success(); - - let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - - // TODO: Don't parse the output, after implementation: - // `kamu log`: support `--output-format json` - // https://github.com/kamu-data/kamu-cli/issues/887 - - stdout - .split("---") - .skip(1) - .map(str::trim) - .map(|block_data| { - let Some(pos) = block_data.find('\n') else { - unreachable!() - }; - let (first_line_with_block_hash, metadata_block_str) = block_data.split_at(pos); - - let block_hash = first_line_with_block_hash - .strip_prefix("# Block: ") - .unwrap(); - let block = YamlMetadataBlockDeserializer {} - .read_manifest(metadata_block_str.as_ref()) - .unwrap(); - - BlockRecord { - block_hash: Multihash::from_multibase(block_hash).unwrap(), - block, - } - }) - .collect() - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -297,8 +254,7 @@ pub struct DatasetRecord { #[serde(rename = "ID")] pub id: DatasetID, pub name: DatasetName, - #[serde(with = "DatasetKindDef")] - pub kind: DatasetKind, + pub kind: String, pub head: Multihash, pub pulled: Option>, pub records: usize, From 94ba337c46d9e5bd4b666b7b4f497061af3ecae2 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 13:15:13 +0200 Subject: [PATCH 13/16] Revert "Merge branch 'master' into chore/push-and-pul-command-e2e-tests" This reverts commit a4ec39c4d5d891aed60b93bb813083a72c958e0c, reversing changes made to 1130fa5bc1f8816d0b2968b1fdb96e135dea55bf. --- CHANGELOG.md | 5 +- Cargo.lock | 146 +++++----- Cargo.toml | 138 ++++----- LICENSE.txt | 2 +- examples/query-commitments/README.md | 17 -- examples/query-commitments/example.py | 266 ------------------ examples/query-commitments/requirements.in | 4 - examples/reth-vs-snp500/init-odf-all.sh | 23 -- examples/reth-vs-snp500/init-odf.sh | 17 -- .../kamu-base-with-data-mt/extra/.kamuconfig | 3 - .../kamu-base-with-data-mt/init-workspace.py | 21 +- 11 files changed, 151 insertions(+), 491 deletions(-) delete mode 100644 examples/query-commitments/README.md delete mode 100755 examples/query-commitments/example.py delete mode 100644 examples/query-commitments/requirements.in delete mode 100755 examples/reth-vs-snp500/init-odf-all.sh delete mode 100755 examples/reth-vs-snp500/init-odf.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c944f6973..4b95638cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Recommendation: for ease of reading, use the following order: - Fixed --> -## [0.204.5] - 2024-10-08 +## [Unreleased] ### Added - Postgres implementation for dataset entry and account Re-BAC repositories - Added (or expanded) E2E tests for: @@ -42,9 +42,6 @@ Recommendation: for ease of reading, use the following order: - Simplified error handling code in repositories - Hidden part of the test code behind the feature gate - Updated our crate dependencies so they can be built in isolation -### Fixed -- `--yes / -y` flag: fixed when working from a TTY -- CI: Fixes `kamu-base-with-data-mt` image builds ## [0.204.4] - 2024-09-30 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 19d059dde4..79f4fb3fd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "async-utils" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", ] @@ -2448,7 +2448,7 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "cfg-if", @@ -2878,7 +2878,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "aws-config", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -3746,7 +3746,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.204.5" +version = "0.204.4" [[package]] name = "env_filter" @@ -3815,7 +3815,7 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -4499,7 +4499,7 @@ dependencies = [ [[package]] name = "http-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "axum", "http 1.1.0", @@ -4798,7 +4798,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "init-on-startup" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -4841,7 +4841,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.204.5" +version = "0.204.4" dependencies = [ "thiserror", ] @@ -4989,7 +4989,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.204.5" +version = "0.204.4" dependencies = [ "alloy", "async-recursion", @@ -5077,7 +5077,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "base32", @@ -5103,7 +5103,7 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5124,7 +5124,7 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5145,7 +5145,7 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5166,7 +5166,7 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "argon2", "chrono", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "argon2", "async-trait", @@ -5209,7 +5209,7 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5230,7 +5230,7 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "dill", @@ -5252,7 +5252,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow-flight", "async-trait", @@ -5275,7 +5275,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-graphql", "async-trait", @@ -5326,7 +5326,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5411,7 +5411,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.204.5" +version = "0.204.4" dependencies = [ "axum", "chrono", @@ -5447,7 +5447,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "internal-error", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common-macros", @@ -5473,7 +5473,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "dill", "kamu-auth-rebac", @@ -5499,7 +5499,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "dill", @@ -5518,7 +5518,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -5535,7 +5535,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow-flight", "async-graphql", @@ -5656,7 +5656,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5678,7 +5678,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -5686,7 +5686,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5699,7 +5699,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5713,7 +5713,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5727,7 +5727,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "indoc 2.0.5", @@ -5745,7 +5745,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5759,7 +5759,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.204.5" +version = "0.204.4" dependencies = [ "assert_cmd", "async-trait", @@ -5775,7 +5775,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -5806,7 +5806,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "arrow-digest", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "async-trait", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.204.5" +version = "0.204.4" dependencies = [ "aes-gcm", "async-trait", @@ -5875,7 +5875,7 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5898,7 +5898,7 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -5935,7 +5935,7 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5966,7 +5966,7 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6018,7 +6018,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6048,7 +6048,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6073,7 +6073,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6086,7 +6086,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6130,7 +6130,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6155,7 +6155,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6191,7 +6191,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6210,7 +6210,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6233,7 +6233,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6247,7 +6247,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6269,7 +6269,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "clap", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6321,7 +6321,7 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6384,7 +6384,7 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6781,7 +6781,7 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6910,7 +6910,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.204.5" +version = "0.204.4" dependencies = [ "base64 0.22.1", "bs58", @@ -7235,7 +7235,7 @@ dependencies = [ [[package]] name = "observability" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "axum", @@ -7294,7 +7294,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "base64 0.22.1", @@ -8254,7 +8254,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.204.5" +version = "0.204.4" dependencies = [ "rand", ] @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "time-source" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -10239,7 +10239,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.204.5" +version = "0.204.4" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index 0aabea069b..9e5404939b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,95 +92,95 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.204.5", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.204.4", path = "src/app/cli", default-features = false } # Utils -async-utils = { version = "0.204.5", path = "src/utils/async-utils", default-features = false } -container-runtime = { version = "0.204.5", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.204.5", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.204.5", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.204.5", path = "src/utils/enum-variants", default-features = false } -event-sourcing = { version = "0.204.5", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.204.5", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.204.5", path = "src/utils/http-common", default-features = false } -init-on-startup = { version = "0.204.5", path = "src/utils/init-on-startup", default-features = false } -internal-error = { version = "0.204.5", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.204.5", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.204.5", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.204.5", path = "src/utils/datafusion-cli", default-features = false } -messaging-outbox = { version = "0.204.5", path = "src/utils/messaging-outbox", default-features = false } -multiformats = { version = "0.204.5", path = "src/utils/multiformats", default-features = false } -observability = { version = "0.204.5", path = "src/utils/observability", default-features = false } -random-names = { version = "0.204.5", path = "src/utils/random-names", default-features = false } -time-source = { version = "0.204.5", path = "src/utils/time-source", default-features = false } -tracing-perfetto = { version = "0.204.5", path = "src/utils/tracing-perfetto", default-features = false } +async-utils = { version = "0.204.4", path = "src/utils/async-utils", default-features = false } +container-runtime = { version = "0.204.4", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.204.4", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.204.4", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.204.4", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.204.4", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.204.4", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.204.4", path = "src/utils/http-common", default-features = false } +init-on-startup = { version = "0.204.4", path = "src/utils/init-on-startup", default-features = false } +internal-error = { version = "0.204.4", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.204.4", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.204.4", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.204.4", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.204.4", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.204.4", path = "src/utils/multiformats", default-features = false } +observability = { version = "0.204.4", path = "src/utils/observability", default-features = false } +random-names = { version = "0.204.4", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.204.4", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.204.4", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.204.5", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.204.5", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.204.5", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.204.5", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.204.5", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.204.5", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.204.5", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.204.4", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.204.4", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.204.4", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.204.4", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.204.4", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.204.4", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.204.4", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.204.5", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.204.5", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.204.5", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.204.5", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.204.5", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.204.4", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.204.4", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.204.4", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.204.4", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.204.4", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.204.5", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.204.5", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.204.4", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.204.4", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.204.5", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.204.5", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.204.5", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.204.5", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.204.4", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.204.4", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.204.4", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.204.4", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.204.5", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.204.5", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.204.5", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.204.5", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.204.5", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.204.4", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.204.4", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.204.4", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.204.4", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.204.4", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.204.5", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.204.5", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.204.5", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.204.5", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.204.4", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.204.4", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.204.4", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.204.4", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.204.5", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.204.5", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.204.5", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.204.5", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.204.4", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.204.4", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.204.4", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.204.4", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.204.5", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.204.5", path = "src/infra/auth-rebac/repo-tests", default-features = false } -kamu-auth-rebac-postgres = { version = "0.204.5", path = "src/infra/auth-rebac/postgres", default-features = false } -kamu-auth-rebac-sqlite = { version = "0.204.5", path = "src/infra/auth-rebac/sqlite", default-features = false } +kamu-auth-rebac-inmem = { version = "0.204.4", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.204.4", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-postgres = { version = "0.204.4", path = "src/infra/auth-rebac/postgres", default-features = false } +kamu-auth-rebac-sqlite = { version = "0.204.4", path = "src/infra/auth-rebac/sqlite", default-features = false } ## Outbox -kamu-messaging-outbox-inmem = { version = "0.204.5", path = "src/infra/messaging-outbox/inmem", default-features = false } -kamu-messaging-outbox-postgres = { version = "0.204.5", path = "src/infra/messaging-outbox/postgres", default-features = false } -kamu-messaging-outbox-sqlite = { version = "0.204.5", path = "src/infra/messaging-outbox/sqlite", default-features = false } -kamu-messaging-outbox-repo-tests = { version = "0.204.5", path = "src/infra/messaging-outbox/repo-tests", default-features = false } +kamu-messaging-outbox-inmem = { version = "0.204.4", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.204.4", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.204.4", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.204.4", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.204.5", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.204.5", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.204.5", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.204.5", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.204.5", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.204.5", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.204.4", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.204.4", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.204.4", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.204.4", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.204.4", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.204.4", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.204.5", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.204.5", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.204.5", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.204.4", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.204.4", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.204.4", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.204.5" +version = "0.204.4" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index 7f5413248c..5a0c90820d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.204.5 +Licensed Work: Kamu CLI Version 0.204.4 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, diff --git a/examples/query-commitments/README.md b/examples/query-commitments/README.md deleted file mode 100644 index 1b83763ae4..0000000000 --- a/examples/query-commitments/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Examples of Verifiable Batch Queries -Query proofs allow you to hold any ODF node forever accountable for the result it provided you, no matter how much time had passed. - -See [batch query commitments documentation](https://docs.kamu.dev/node/commitments/) and [REST API reference](https://docs.kamu.dev/node/protocols/rest-api/) for the overview of this mechanism. - -The included script illustrates: -- how to query data and receive a cryptographic proof -- how to validate commitment consistency on the client side -- and how to ask another node to verify the commitment by reproducing the query. - -Running: -```sh -pip install -r requirements.in -python ./example.py --node-url https://node.example.com --private-key -``` - -Private key is used to show examples of failed verification (by forging a signature). If you don't provide it - those cases will be skipped. diff --git a/examples/query-commitments/example.py b/examples/query-commitments/example.py deleted file mode 100755 index 5a2b78e6ea..0000000000 --- a/examples/query-commitments/example.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python -import argparse -import copy -import canonicaljson -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey -import hashlib -import requests -import json -import base64 -import base58 - - -def decode_multibase(multibase): - prefix, value = multibase[0], multibase[1:] - if prefix == 'u': - b64_urlsafe_nopad = value - b64_urlsafe = b64_urlsafe_nopad + "=" * ((4 - len(b64_urlsafe_nopad) % 4) % 4) - return base64.urlsafe_b64decode(b64_urlsafe) - elif prefix == 'f': - return bytes.fromhex(value) - elif prefix == 'z': - return base58.b58decode(value) - else: - raise("Malformed multibase value") - - -def encode_multibase_base64_urlsafe_nopad(bin): - return 'u' + base64.urlsafe_b64encode(bin).decode('utf-8').rstrip("=") - - -def multihash_sha3_256_multibase_base16(bin): - return 'f1620' + hashlib.sha3_256(bin).hexdigest() - - -def public_key_from_did(did): - assert did.startswith('did:key:') - multicodec = decode_multibase(did[len('did:key:'):]) - assert multicodec[0:1].hex() == 'ed' # 0xed is multicodec value for Ed25519Pub - return Ed25519PublicKey.from_public_bytes(multicodec[2:]) - -def main(args): - # Query data - resp = requests.get( - args.node_url + "/query", - params=dict( - query=args.query, - include="proof", - ) - ) - resp.raise_for_status() - resp_data = resp.json() - - print(">>> Node's response:") - print(json.dumps(resp_data, indent=2)) - print() - print() - - - # Verify commitment consistency - # This should always be done by the client after receiving a proof to ensure its disputable - assert resp_data["commitment"]["inputHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["input"]) - ) - assert resp_data["commitment"]["outputHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["output"]) - ) - assert resp_data["commitment"]["subQueriesHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["subQueries"]) - ) - - signature = decode_multibase(resp_data["proof"]["proofValue"]) - public_key = public_key_from_did(resp_data["proof"]["verificationMethod"]) - public_key.verify(signature, canonicaljson.encode_canonical_json(resp_data["commitment"])) - print("Commitment is consistent!") - - - print(">>> Commitment:") - commitment = resp_data.copy() - del commitment["output"] - print(json.dumps(commitment, indent=2)) - print() - print() - - - # Remote validation through reproducibility - print(">>> Verifying original commitment:") - resp = requests.post( - args.node_url + "/verify", - json=commitment - ) - resp.raise_for_status() - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == True - - - # Invalid request: input hash - print(">>> Simulating invalid request (input hash):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["inputHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::InputHash" - - - # Invalid request: subQueries hash - print(">>> Simulating invalid request (subQueries hash):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["subQueriesHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::SubQueriesHash" - - - # Invalid request: bad signature - print(">>> Simulating invalid request (bad signature):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["proof"]["proofValue"] = "uZbm7fFcWc4l6iyvaKe_txdKntL3h3kvsGHOaKIbPV6c42PH1VnSmpYHMopv4TU68syzgoEdcS26AvpkSQb9dBQ" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::BadSignature" - - - if args.private_key is None: - print("Private key is not provided - skipping tests that require signature forging") - return - - private_key = Ed25519PrivateKey.from_private_bytes(decode_multibase(args.private_key)) - - - # Cannot reproduce the query: output mismatch - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (output mismatch):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["outputHash"] = "f1620ff7f5beaf16900218a3ac4aae82cdccf764816986c7c739c716cf7dc03112a2d" - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::OutputMismatch" - - - # Cannot reproduce the query: dataset is missing - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (dataset is missing):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["input"]["datasets"][0]["id"] = invalid_commitment["input"]["datasets"][0]["id"][:-4] + "beef" - invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json( - invalid_commitment["input"] - ) - ) - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::DatasetNotFound" - - - # Cannot reproduce the query: block is missing - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (block is missing):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["input"]["datasets"][0]["blockHash"] = invalid_commitment["input"]["datasets"][0]["blockHash"][:-4] + "beef" - invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json( - invalid_commitment["input"] - ) - ) - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::DatasetBlockNotFound" - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--node-url', required=True) - parser.add_argument('--private-key', required=False) - parser.add_argument('--query', default='select block_hash, to from "kamu/net.rocketpool.reth.tokens-minted" order by offset desc limit 1') - args = parser.parse_args() - main(args) \ No newline at end of file diff --git a/examples/query-commitments/requirements.in b/examples/query-commitments/requirements.in deleted file mode 100644 index ab209766b2..0000000000 --- a/examples/query-commitments/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -base58 -canonicaljson -cryptography -requests \ No newline at end of file diff --git a/examples/reth-vs-snp500/init-odf-all.sh b/examples/reth-vs-snp500/init-odf-all.sh deleted file mode 100755 index 1739ff3877..0000000000 --- a/examples/reth-vs-snp500/init-odf-all.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -set -e - -NODE_URL="odf+https://node.demo.kamu.dev/kamu/" - -kamu init || true - -# Root -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" -kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" -kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" - -kamu pull "${NODE_URL}account.transactions" -kamu pull "${NODE_URL}account.tokens.transfers" - -# Deriv -kamu pull "${NODE_URL}net.rocketpool.reth.mint-burn" -kamu pull "${NODE_URL}account.tokens.portfolio" -kamu pull "${NODE_URL}account.tokens.portfolio.market-value" -kamu pull "${NODE_URL}account.tokens.portfolio.usd" -kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.market-value" -kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.portfolio" diff --git a/examples/reth-vs-snp500/init-odf.sh b/examples/reth-vs-snp500/init-odf.sh deleted file mode 100755 index abec551cce..0000000000 --- a/examples/reth-vs-snp500/init-odf.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -set -e - -NODE_URL="odf+https://node.demo.kamu.dev/kamu/" - -kamu init || true - -# Root -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" -kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" -kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" - -kamu pull "${NODE_URL}account.transactions" -kamu pull "${NODE_URL}account.tokens.transfers" - -kamu add -r . diff --git a/images/kamu-base-with-data-mt/extra/.kamuconfig b/images/kamu-base-with-data-mt/extra/.kamuconfig index 68000e8f80..0462429714 100644 --- a/images/kamu-base-with-data-mt/extra/.kamuconfig +++ b/images/kamu-base-with-data-mt/extra/.kamuconfig @@ -3,9 +3,6 @@ version: 1 content: users: predefined: - - accountName: kamu - isAdmin: true - avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4 - accountName: sh101-bowen avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png - accountName: sh102-gambier diff --git a/images/kamu-base-with-data-mt/init-workspace.py b/images/kamu-base-with-data-mt/init-workspace.py index 286c24ce44..1d2b50a361 100755 --- a/images/kamu-base-with-data-mt/init-workspace.py +++ b/images/kamu-base-with-data-mt/init-workspace.py @@ -1,55 +1,48 @@ #!/usr/bin/env python -import shutil +import os +import sys import subprocess -from pathlib import Path ############################################################################### -CURRENT_PATH = Path(__file__).resolve().parent S3_REPO_URL = "s3://datasets.kamu.dev/odf/v2/example-mt/" - ############################################################################### def s3_listdir(url): return [ line.strip().split(' ')[1] for line in subprocess.run( - f"aws s3 ls {url}", - shell=True, + f"aws s3 ls {url}", + shell=True, text=True, check=True, capture_output=True, ).stdout.splitlines() ] - def s3_cat(url): return subprocess.run( - f"aws s3 cp {url} -", - shell=True, + f"aws s3 cp {url} -", + shell=True, text=True, check=True, capture_output=True, ).stdout.strip() - ############################################################################### subprocess.run( - "kamu init --multi-tenant --exists-ok", + "kamu init --multi-tenant --exists-ok", shell=True, check=True, ) -shutil.copy(CURRENT_PATH / "extra/.kamuconfig", ".kamuconfig") - for did in s3_listdir(S3_REPO_URL): url = S3_REPO_URL + did alias = s3_cat(f"{S3_REPO_URL}{did}info/alias") account, name = alias.split('/', 1) - subprocess.run( f"kamu --account {account} pull --no-alias {url} --as {name}", shell=True, From ec01c6fe1eaa4274fdbc9bdf874a06ae5af9bf65 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 13:29:39 +0200 Subject: [PATCH 14/16] Clean code --- .../repo-tests/src/test_smart_transfer_protocol.rs | 14 +++++++------- .../kamu-cli-puppet/src/kamu_cli_puppet_ext.rs | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 3d7b5a6b34..09cb6009e3 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -1161,13 +1161,13 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { let expected_schema = indoc::indoc!( r#" message arrow_schema { - REQUIRED INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; + REQUIRED INT64 offset; + REQUIRED INT32 op; + REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); + OPTIONAL INT64 match_id; + OPTIONAL BYTE_ARRAY player_id (STRING); + OPTIONAL INT64 score; } "# ); diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 0abaddb68e..438ef28a5d 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -67,7 +67,6 @@ impl KamuCliPuppetExt for KamuCliPuppet { .success(); let stdout = std::str::from_utf8(&assert.get_output().stdout).unwrap(); - println!("@@@@@ stdout: {:?}", stdout); serde_json::from_str(stdout).unwrap() } From 159ff461ccff5f4f22db94362ca8cf2a3e6076a9 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 14:21:26 +0200 Subject: [PATCH 15/16] Fix review comments. Iter 2 --- .../common/src/kamu_api_server_client_ext.rs | 5 -- .../tests/test_smart_transfer_protocol.rs | 4 ++ .../src/test_smart_transfer_protocol.rs | 65 +++++++++---------- .../src/kamu_cli_puppet_ext.rs | 2 + 4 files changed, 37 insertions(+), 39 deletions(-) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index bee28b2b52..408f9ac53c 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -260,11 +260,6 @@ impl KamuApiServerClientExt for KamuApiServerClient { account_name_maybe: Option, ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; - - // TODO: Use the alias from the reply, after fixing the bug: - // https://github.com/kamu-data/kamu-cli/issues/891 - - // At the moment, only single-tenant let dataset_alias = DatasetAlias::new( account_name_maybe, DatasetName::new_unchecked("player-scores"), diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index c99f906c05..abb5120d31 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -83,6 +83,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, + options = Options::default().with_frozen_system_time(), ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -90,6 +91,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -109,6 +111,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -117,6 +120,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 09cb6009e3..5deee2f3e6 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -452,7 +452,7 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien // 2. Pushing datasets to the API server { kamu_in_push_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // 2.1. Add datasets { @@ -566,8 +566,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -590,8 +590,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -659,8 +659,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -669,8 +669,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -707,7 +707,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe // 2. Pushing datasets to the API server { kamu_in_push_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // 2.1. Add datasets { @@ -764,7 +764,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe { let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; kamu_in_pull_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // Pull datasets one by one and check data run_and_assert_command_success( @@ -807,8 +807,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -831,8 +831,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -882,8 +882,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -892,8 +892,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -925,7 +925,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { "pull", dataset_name.as_str(), "--set-watermark", - "2000-01-01T00:00:00Z", + "2051-01-02T03:04:05Z", ]) .await .success(); @@ -940,8 +940,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); @@ -988,8 +987,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -1043,8 +1042,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -1123,8 +1122,7 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) @@ -1176,8 +1174,8 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -1188,8 +1186,7 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); @@ -1249,8 +1246,8 @@ pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 438ef28a5d..bebf25b1e7 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -253,6 +253,8 @@ pub struct DatasetRecord { #[serde(rename = "ID")] pub id: DatasetID, pub name: DatasetName, + // CLI returns regular ENUM DatasetKind(Root/Derivative) for local datasets + // but for remote it is Remote(DatasetKind) type pub kind: String, pub head: Multihash, pub pulled: Option>, From 87f26049be6bfb05c76acb694adba3e7e87a3557 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 14:45:02 +0200 Subject: [PATCH 16/16] Revert ToDo comment --- src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index 408f9ac53c..ef4fc862e3 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -260,6 +260,9 @@ impl KamuApiServerClientExt for KamuApiServerClient { account_name_maybe: Option, ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; + + // TODO: Use the alias from the reply, after fixing the bug: + // https://github.com/kamu-data/kamu-cli/issues/891 let dataset_alias = DatasetAlias::new( account_name_maybe, DatasetName::new_unchecked("player-scores"),