From bbb8eebc0b22965893752e5aea8f9f49f11dbf2e Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 25 Oct 2021 11:11:21 +0800 Subject: [PATCH 001/111] add delegation verification tool --- .../delegation_verify/delegation_verify.ml | 192 ++++++++++++++++++ .../delegation_verify/delegation_verify.opam | 5 + src/app/delegation_verify/dune | 4 + 3 files changed, 201 insertions(+) create mode 100644 src/app/delegation_verify/delegation_verify.ml create mode 100644 src/app/delegation_verify/delegation_verify.opam create mode 100644 src/app/delegation_verify/dune diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml new file mode 100644 index 000000000000..6d5f9d4cc8f1 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.ml @@ -0,0 +1,192 @@ +open Mina_base +open Mina_transition +open Core +open Async +open Signature_lib + +type metadata = + { submitted_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + } +[@@deriving yojson] + +let get_filenames = + let open In_channel in + function + | [ "-" ] | [] -> + input_all stdin |> String.split_lines + | filenames -> + filenames + +(* This check seems unnecessary if the submission data is published by ourselfes *) +let check_path str = + match String.split str ~on:'/' with + | [ _basedir; submitter; block_hash; created_at ] -> ( + match submitter |> Yojson.Safe.from_string |> Public_key.of_yojson with + | Ok submitter -> ( + match Ptime.of_rfc3339 created_at with + | Ok (_, _, _) -> + Ok (submitter, block_hash) + | Error _ -> + Error `Path_is_invalid ) + | Error _ -> + Error `Path_is_invalid ) + | _ -> + Error `Path_is_invalid + +let load_metadata str = + try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata + +(* This decoding is also unnecessarily complicated given that we are the creator of those data *) +let decode_metadata str = + match Yojson.Safe.from_string str |> metadata_of_yojson with + | Ok { submitted_at = _; peer_id = _; snark_work; remote_addr = _ } -> + Ok snark_work + | Error _ -> + Error `Fail_to_decode_metadata + +let load_block ~block_dir ~block_hash = + let block_path = Filename.concat block_dir (block_hash ^ ".dat") in + try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block + +let decode_block str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + with _ -> Error `Fail_to_decode_block ) + | Error _ -> + Error `Fail_to_decode_block + +let verify_block ~verifier ~block = + let open External_transition in + match%map + Verifier.verify_blockchain_snarks verifier + [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) + ~proof:(protocol_state_proof block) + ] + with + | Ok result -> + if result then Ok () else Error `Invalid_proof + | Error e -> + Error (`Verifier_error e) + +let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error `Fail_to_decode_snark_work ) + | Error _ -> + Error `Fail_to_decode_snark_work + +let verify_snark_work ~verifier ~proof ~message = + match%map + Verifier.verify_transaction_snarks verifier [ (proof, message) ] + with + | Ok true -> + Ok () + | Ok false -> + Error `Invalid_snark_work + | Error e -> + Error (`Verifier_error e) + +let validate_submission ~verifier ~block_dir ~metadata_path = + let open Deferred.Result.Let_syntax in + let%bind submitter, block_hash = + Deferred.return @@ check_path metadata_path + in + let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in + let%bind snark_work_opt = Deferred.return @@ decode_metadata metadata_str in + let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in + let%bind block = Deferred.return @@ decode_block block_str in + let%bind () = verify_block ~verifier ~block in + let%map () = + match snark_work_opt with + | None -> + Deferred.Result.return () + | Some snark_work_str -> + let%bind Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } = + Deferred.return @@ decode_snark_work snark_work_str + in + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:(Public_key.compress submitter) + in + verify_snark_work ~verifier ~proof ~message + in + ( External_transition.state_hash block + , External_transition.blockchain_length block + , External_transition.global_slot block ) + +type valid_payload = + { state_hash : State_hash.t + ; height : Unsigned.uint32 + ; slot : Unsigned.uint32 + } + +let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = + `Assoc + [ ("state_hash", State_hash.to_yojson state_hash) + ; ("height", `Int (Unsigned.UInt32.to_int height)) + ; ("slot", `Int (Unsigned.UInt32.to_int slot)) + ] + +let display valid_payload = + printf "%s" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload + +let display_error e = + eprintf "%s" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] + +let command = + Command.async + ~summary:"A tool for verifying JSON payload submitted by the uptime service" + Command.Let_syntax.( + let%map_open block_dir = + flag "--block-dir" ~aliases:[ "-block-dir" ] + ~doc:"the path to the directory containing blocks for the submission" + (required Filename.arg_type) + and inputs = anon (sequence ("filename" %: Filename.arg_type)) in + fun () -> + let open Deferred.Let_syntax in + let logger = Logger.create () in + let%bind verifier = + Verifier.create ~logger + ~proof_level:Genesis_constants.Proof_level.compiled + ~constraint_constants: + Genesis_constants.Constraint_constants.compiled + ~pids:(Pid.Table.create ()) ~conf_dir:None + in + let metadata_pathes = get_filenames inputs in + Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> + match%bind + validate_submission ~verifier ~block_dir ~metadata_path + with + | Ok (state_hash, height, slot) -> + display { state_hash; height; slot } ; + Deferred.unit + | Error `Path_is_invalid -> + display_error "path for metadata is invalid" ; + exit 1 + | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> + display_error "fail to load metadata" ; + exit 2 + | Error `Fail_to_load_block | Error `Fail_to_decode_block -> + display_error "fail to load block" ; + exit 3 + | Error `Invalid_proof -> + display_error + "fail to verify the protocol state proof inside the block" ; + exit 5 + | Error (`Verifier_error e) -> + display_error @@ "verifier error: " ^ Error.to_string_hum e ; + exit 5 + | Error `Fail_to_decode_snark_work -> + display_error "fail to decode snark work" ; + exit 5 + | Error `Invalid_snark_work -> + display_error "fail to verify the snark work" ; + exit 5)) + +let () = Command.run command diff --git a/src/app/delegation_verify/delegation_verify.opam b/src/app/delegation_verify/delegation_verify.opam new file mode 100644 index 000000000000..7be19e3d6129 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.opam @@ -0,0 +1,5 @@ +opam-version: "2.0" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune new file mode 100644 index 000000000000..c683a012b03d --- /dev/null +++ b/src/app/delegation_verify/dune @@ -0,0 +1,4 @@ +(executable + (name delegation_verify) + (libraries uptime_service core_kernel async mina_transition) + (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) \ No newline at end of file From c9a09382273dc16b2b9d2e588dd72b24ec1dee14 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 25 Oct 2021 11:19:05 +0800 Subject: [PATCH 002/111] returns 0 even if the verification fails --- src/app/delegation_verify/delegation_verify.ml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6d5f9d4cc8f1..951030d0a8b2 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -178,15 +178,15 @@ let command = | Error `Invalid_proof -> display_error "fail to verify the protocol state proof inside the block" ; - exit 5 + Deferred.unit | Error (`Verifier_error e) -> display_error @@ "verifier error: " ^ Error.to_string_hum e ; - exit 5 + Deferred.unit | Error `Fail_to_decode_snark_work -> display_error "fail to decode snark work" ; - exit 5 + Deferred.unit | Error `Invalid_snark_work -> display_error "fail to verify the snark work" ; - exit 5)) + Deferred.unit)) let () = Command.run command From c5f039cab51de33658ac77f018d6457be5388abc Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 27 Oct 2021 03:23:25 +0800 Subject: [PATCH 003/111] adopt to new path structure for delegation data --- .../delegation_verify/delegation_verify.ml | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 951030d0a8b2..dcf84d84da1a 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -5,10 +5,12 @@ open Async open Signature_lib type metadata = - { submitted_at : string + { created_at : string ; peer_id : string ; snark_work : string option [@default None] ; remote_addr : string + ; submitter : string + ; block_hash : string } [@@deriving yojson] @@ -21,20 +23,7 @@ let get_filenames = filenames (* This check seems unnecessary if the submission data is published by ourselfes *) -let check_path str = - match String.split str ~on:'/' with - | [ _basedir; submitter; block_hash; created_at ] -> ( - match submitter |> Yojson.Safe.from_string |> Public_key.of_yojson with - | Ok submitter -> ( - match Ptime.of_rfc3339 created_at with - | Ok (_, _, _) -> - Ok (submitter, block_hash) - | Error _ -> - Error `Path_is_invalid ) - | Error _ -> - Error `Path_is_invalid ) - | _ -> - Error `Path_is_invalid +let check_path _ = Ok () let load_metadata str = try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata @@ -42,8 +31,19 @@ let load_metadata str = (* This decoding is also unnecessarily complicated given that we are the creator of those data *) let decode_metadata str = match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ok { submitted_at = _; peer_id = _; snark_work; remote_addr = _ } -> - Ok snark_work + | Ok + { created_at = _ + ; peer_id = _ + ; snark_work + ; remote_addr = _ + ; submitter + ; block_hash + } -> ( + match submitter |> Public_key.Compressed.of_base58_check with + | Ok submitter -> + Ok (snark_work, submitter, block_hash) + | Error _ -> + Error `Fail_to_decode_metadata ) | Error _ -> Error `Fail_to_decode_metadata @@ -52,12 +52,8 @@ let load_block ~block_dir ~block_hash = try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block let decode_block str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module External_transition.Stable.Latest) str) - with _ -> Error `Fail_to_decode_block ) - | Error _ -> - Error `Fail_to_decode_block + try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + with _ -> Error `Fail_to_decode_block let verify_block ~verifier ~block = let open External_transition in @@ -93,11 +89,11 @@ let verify_snark_work ~verifier ~proof ~message = let validate_submission ~verifier ~block_dir ~metadata_path = let open Deferred.Result.Let_syntax in - let%bind submitter, block_hash = - Deferred.return @@ check_path metadata_path - in + let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in - let%bind snark_work_opt = Deferred.return @@ decode_metadata metadata_str in + let%bind snark_work_opt, submitter, block_hash = + Deferred.return @@ decode_metadata metadata_str + in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in let%bind () = verify_block ~verifier ~block in @@ -111,8 +107,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = Deferred.return @@ decode_snark_work snark_work_str in let message = - Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:(Public_key.compress submitter) + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in verify_snark_work ~verifier ~proof ~message in @@ -156,7 +151,8 @@ let command = ~proof_level:Genesis_constants.Proof_level.compiled ~constraint_constants: Genesis_constants.Constraint_constants.compiled - ~pids:(Pid.Table.create ()) ~conf_dir:None + ~pids:(Child_processes.Termination.create_pid_table ()) + ~conf_dir:None in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> @@ -189,4 +185,4 @@ let command = display_error "fail to verify the snark work" ; Deferred.unit)) -let () = Command.run command +let () = Rpc_parallel.start_app command From 872c2996ac033749b8d55db2b0acc61e86ffc597 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 27 Oct 2021 03:37:33 +0800 Subject: [PATCH 004/111] disable logs --- src/app/delegation_verify/delegation_verify.ml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index dcf84d84da1a..357fbdc1fbea 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -129,10 +129,11 @@ let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = ] let display valid_payload = - printf "%s" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload + printf "%s\n" @@ Yojson.Safe.to_string + @@ valid_payload_to_yojson valid_payload let display_error e = - eprintf "%s" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] + eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] let command = Command.async @@ -145,7 +146,7 @@ let command = and inputs = anon (sequence ("filename" %: Filename.arg_type)) in fun () -> let open Deferred.Let_syntax in - let logger = Logger.create () in + let logger = Logger.null () in let%bind verifier = Verifier.create ~logger ~proof_level:Genesis_constants.Proof_level.compiled From 2c5e3545ed6795865d77e58b802b3ca57d072900 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 01:30:50 +0800 Subject: [PATCH 005/111] adding dockerfiles for stateless verification tool --- dockerfiles/stages/4-stateless-verify | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 dockerfiles/stages/4-stateless-verify diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/stages/4-stateless-verify new file mode 100644 index 000000000000..bbefc57e2746 --- /dev/null +++ b/dockerfiles/stages/4-stateless-verify @@ -0,0 +1,54 @@ +################################################################################################# +# The "stateless verification build" Stage +# - builds stateless verification tool +# - should not include any data related to joining a specific network, only the node software itself +################################################################################################# +FROM opam-deps AS builder + +# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI +ARG DUNE_PROFILE=devnet + +# branch to checkout on first clone (this will be the only availible branch in the container) +# can also be a tagged release +ARG MINA_BRANCH=compatible + +# repo to checkout the branch from +ARG MINA_REPO=https://github.com/MinaProtocol/mina + +# location of repo used for pins and external package commits +ARG MINA_DIR=mina + +ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" + +# git will clone into an empty dir, but this also helps us set the workdir in advance +RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ + && git clone \ + -b "${MINA_BRANCH}" \ + --depth 1 \ + --shallow-submodules \ + --recurse-submodules \ + ${MINA_REPO} ${HOME}/${MINA_DIR} + +WORKDIR $HOME/${MINA_DIR} + +RUN mkdir ${HOME}/app + +# Build libp2p_helper +RUN make libp2p_helper \ + && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper + +# HACK: build without special cpu features to allow more people to run mina-rosetta +RUN ./scripts/zexe-standardize.sh + +# Make rosetta-crucial components and the generate_keypair tool +RUN eval $(opam config env) \ + && dune build --profile=${DUNE_PROFILE} \ + src/app/delegation_verify.exe \ + && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ + && rm -rf _build + +# Clear go module caches +RUN cd src/app/libp2p_helper/src/libp2p_helper \ + && go clean --cache --modcache --testcache -r + +ENTRYPOINT ["delegation-verify"] \ No newline at end of file From 6bf8a3c12cfaeaa2fe3e778287ed2916cf3c75ab Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 02:13:09 +0800 Subject: [PATCH 006/111] add delegation-verify service --- scripts/release-docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 1c69f91c0b89..cd4fbdd12df4 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -134,6 +134,9 @@ itn-orchestrator) DOCKER_CONTEXT="src/app/itn_orchestrator" ;; +delegation-verify) + DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-toolchain dockerfiles/stages/3-opam-deps dockerfiles/stages/4-stateless-verify" + ;; esac From e04dda9f5010bf52f8e209b41ce1e45ddf5d384b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 19:11:37 +0800 Subject: [PATCH 007/111] remote libp2p&go stuff in docker file --- dockerfiles/stages/4-stateless-verify | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/stages/4-stateless-verify index bbefc57e2746..b8a5e09b54c7 100644 --- a/dockerfiles/stages/4-stateless-verify +++ b/dockerfiles/stages/4-stateless-verify @@ -33,22 +33,13 @@ WORKDIR $HOME/${MINA_DIR} RUN mkdir ${HOME}/app -# Build libp2p_helper -RUN make libp2p_helper \ - && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper - -# HACK: build without special cpu features to allow more people to run mina-rosetta +# HACK: build without special cpu features to allow more people to run delegation verification tool RUN ./scripts/zexe-standardize.sh -# Make rosetta-crucial components and the generate_keypair tool RUN eval $(opam config env) \ && dune build --profile=${DUNE_PROFILE} \ src/app/delegation_verify.exe \ && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ && rm -rf _build -# Clear go module caches -RUN cd src/app/libp2p_helper/src/libp2p_helper \ - && go clean --cache --modcache --testcache -r - ENTRYPOINT ["delegation-verify"] \ No newline at end of file From e5fe67271a97a33a4c45345e886aedc1205bb8a8 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 19:20:50 +0800 Subject: [PATCH 008/111] add readme --- src/app/delegation_verify/README.md | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/app/delegation_verify/README.md diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md new file mode 100644 index 000000000000..806ebcd85504 --- /dev/null +++ b/src/app/delegation_verify/README.md @@ -0,0 +1,58 @@ +This PR implements the stateless verification tool that would be used to verify that data collected by the uptime service. + +Usage: +``` +./delegation-verify --block-dir ... +./delegation-verify --block-dir - +``` +Where `` is a file path of the form `//-.json` +containing JSON following the format described in [Cloud storage section of delegation backend spec](https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage) (`` is a place holder for some directory): + +```json +{ "created_at": "server's timestamp (of the time of submission)" +, "remote_addr": "ip:port address from which request has come" +, "peer_id": "peer id (as in user's JSON submission)" +, "snark_work": "(optional) base64-encoded snark work blob" +, "block_hash": "base58check-encoded hash of a block" +, "submitter": "base58check-encoded submitter's public key" +} +``` + +File path content: +- `submitted_at_date` with server's date (of the time of submission) in format `YYYY-MM-DD` +- `submitted_at` with server's timestamp (of the time of submission) in RFC-3339 +- `submitter` is base58check-encoded submitter's public key + + +Parameter `--block-dir` specifies the path to the directory containing block for the submission. It is expected that blocks with `` exist under path `/.dat` for all of the filepaths provided. + +When `-` symbol is used as the `filename1`, no other `filename{i}` are accepted and all filenames will be read from the `stdin`, one filename per line. + +Utility `./delegation-verify` exits with the following exit codes: + +- `0` if it was able to execute verification of all filepaths + - Note that `0` is returned even when some of the payloads were marked as invalid +- `1` if one of `` files violated the naming convention +- `2` if the utility failed to read some of `` files +- `3` if the utility failed to read some of `/.dat` files +- `5` if the other error occurred + +In case of non-zero exit code, some details of the error may be printed to `stderr`. + +For each `` parameter one line of output is printed to `stdout` : + +- Valid payload at `` : + + ```json + { "state_hash": "" + , "height": "" + , "slot": "" + } + ``` + +- Invalid payload at `` : + + ```json + { "error": "" } + ``` +The stateless verification would check the protocol_state_proof in the block and the transaction_snark_proof in the snark_work. \ No newline at end of file From 3821572d7c79c7e556b7e1c5386f85b373e91299 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 21:13:19 +0800 Subject: [PATCH 009/111] add delegation-verify as valid service --- scripts/release-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index cd4fbdd12df4..44c75bce1523 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -10,7 +10,7 @@ set +x CLEAR='\033[0m' RED='\033[0;31m' # Array of valid service names -VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'itn-orchestrator') +VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'delegation-verify' ) function usage() { if [[ -n "$1" ]]; then From f367386c5095a6c80cf5225530c81facbc870740 Mon Sep 17 00:00:00 2001 From: georgeee Date: Thu, 28 Oct 2021 20:32:15 +0300 Subject: [PATCH 010/111] Use base image from URL --- ...ss-verify => Dockerfile-delegation-stateless-verifier} | 8 ++++---- scripts/release-docker.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename dockerfiles/{stages/4-stateless-verify => Dockerfile-delegation-stateless-verifier} (82%) diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/Dockerfile-delegation-stateless-verifier similarity index 82% rename from dockerfiles/stages/4-stateless-verify rename to dockerfiles/Dockerfile-delegation-stateless-verifier index b8a5e09b54c7..8a6adc76f6e5 100644 --- a/dockerfiles/stages/4-stateless-verify +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -3,7 +3,7 @@ # - builds stateless verification tool # - should not include any data related to joining a specific network, only the node software itself ################################################################################################# -FROM opam-deps AS builder +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:7ef5827fa0854a0bcfdee69dcc0c2c7aef86e1662c630e71f07c1f9162e757fa AS builder # Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI ARG DUNE_PROFILE=devnet @@ -38,8 +38,8 @@ RUN ./scripts/zexe-standardize.sh RUN eval $(opam config env) \ && dune build --profile=${DUNE_PROFILE} \ - src/app/delegation_verify.exe \ - && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ + src/app/delegation_verify/delegation_verify.exe \ + && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ && rm -rf _build -ENTRYPOINT ["delegation-verify"] \ No newline at end of file +ENTRYPOINT ["./delegation-verify"] diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 44c75bce1523..12db91c57283 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -135,7 +135,7 @@ itn-orchestrator) ;; delegation-verify) - DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-toolchain dockerfiles/stages/3-opam-deps dockerfiles/stages/4-stateless-verify" + DOCKERFILE_PATH="dockerfiles/Dockerfile-delegation-stateless-verifier" ;; esac From 81da07c0bf9c3552b26581f474a031c0bc6e8d26 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 1 Nov 2021 13:13:08 +0800 Subject: [PATCH 011/111] inline verifier --- .../delegation_verify/delegation_verify.ml | 55 ++++++------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 357fbdc1fbea..0541bb85d6d4 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -55,18 +55,15 @@ let decode_block str = try Ok (Binable.of_string (module External_transition.Stable.Latest) str) with _ -> Error `Fail_to_decode_block -let verify_block ~verifier ~block = +let verify_block ~block = let open External_transition in - match%map - Verifier.verify_blockchain_snarks verifier + let%map result = + Verifier.verify_blockchain_snarks [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) ~proof:(protocol_state_proof block) ] - with - | Ok result -> - if result then Ok () else Error `Invalid_proof - | Error e -> - Error (`Verifier_error e) + in + if result then Ok () else Error `Invalid_proof let decode_snark_work str = match Base64.decode str with @@ -76,18 +73,11 @@ let decode_snark_work str = | Error _ -> Error `Fail_to_decode_snark_work -let verify_snark_work ~verifier ~proof ~message = - match%map - Verifier.verify_transaction_snarks verifier [ (proof, message) ] - with - | Ok true -> - Ok () - | Ok false -> - Error `Invalid_snark_work - | Error e -> - Error (`Verifier_error e) - -let validate_submission ~verifier ~block_dir ~metadata_path = +let verify_snark_work ~proof ~message = + let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in + if result then Ok () else Error `Invalid_snark_work + +let validate_submission ~block_dir ~metadata_path = let open Deferred.Result.Let_syntax in let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in @@ -96,7 +86,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = verify_block ~verifier ~block in + let%bind () = verify_block ~block in let%map () = match snark_work_opt with | None -> @@ -109,7 +99,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = let message = Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in - verify_snark_work ~verifier ~proof ~message + verify_snark_work ~proof ~message in ( External_transition.state_hash block , External_transition.blockchain_length block @@ -146,20 +136,9 @@ let command = and inputs = anon (sequence ("filename" %: Filename.arg_type)) in fun () -> let open Deferred.Let_syntax in - let logger = Logger.null () in - let%bind verifier = - Verifier.create ~logger - ~proof_level:Genesis_constants.Proof_level.compiled - ~constraint_constants: - Genesis_constants.Constraint_constants.compiled - ~pids:(Child_processes.Termination.create_pid_table ()) - ~conf_dir:None - in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> - match%bind - validate_submission ~verifier ~block_dir ~metadata_path - with + match%bind validate_submission ~block_dir ~metadata_path with | Ok (state_hash, height, slot) -> display { state_hash; height; slot } ; Deferred.unit @@ -169,16 +148,16 @@ let command = | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> display_error "fail to load metadata" ; exit 2 - | Error `Fail_to_load_block | Error `Fail_to_decode_block -> + | Error `Fail_to_load_block -> display_error "fail to load block" ; exit 3 + | Error `Fail_to_decode_block -> + display_error "fail to decode block" ; + Deferred.unit | Error `Invalid_proof -> display_error "fail to verify the protocol state proof inside the block" ; Deferred.unit - | Error (`Verifier_error e) -> - display_error @@ "verifier error: " ^ Error.to_string_hum e ; - Deferred.unit | Error `Fail_to_decode_snark_work -> display_error "fail to decode snark work" ; Deferred.unit From 73bd8d8b9f5d365896bb90b5b346881d5eb006dc Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 1 Nov 2021 13:14:07 +0800 Subject: [PATCH 012/111] inline verifier --- src/app/delegation_verify/verifier.ml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/app/delegation_verify/verifier.ml diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml new file mode 100644 index 000000000000..aa95cebf85f0 --- /dev/null +++ b/src/app/delegation_verify/verifier.ml @@ -0,0 +1,23 @@ +open Core + +module T = Transaction_snark.Make (struct + let constraint_constants = Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled +end) + +module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled +end) + +let verify_blockchain_snarks (chains : Blockchain_snark.Blockchain.t list) = + B.Proof.verify + @@ List.map chains ~f:(fun snark -> + ( Blockchain_snark.Blockchain.state snark + , Blockchain_snark.Blockchain.proof snark )) + +let verify_transaction_snarks ts = T.verify ts From 9514c14f72f1528489b7267aaed7cac6baae836b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Tue, 2 Nov 2021 14:38:10 +0800 Subject: [PATCH 013/111] adding bisect_ppx --- src/app/delegation_verify/dune | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index c683a012b03d..7df3a9b2c5dd 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,4 +1,5 @@ (executable (name delegation_verify) - (libraries uptime_service core_kernel async mina_transition) - (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) \ No newline at end of file + (libraries uptime_service core_kernel async mina_transition parallel) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) From b600f7ab6b57cf380ac0d614087715d999b9e0df Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Tue, 25 Jan 2022 19:44:14 +0800 Subject: [PATCH 014/111] adding an option "no-checks" to bypass all the checks --- .../delegation_verify/delegation_verify.ml | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 0541bb85d6d4..4e2c8d78afae 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -77,7 +77,7 @@ let verify_snark_work ~proof ~message = let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work -let validate_submission ~block_dir ~metadata_path = +let validate_submission ~block_dir ~metadata_path ~no_checks = let open Deferred.Result.Let_syntax in let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in @@ -86,34 +86,40 @@ let validate_submission ~block_dir ~metadata_path = in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = verify_block ~block in + let%bind () = if no_checks then return () else verify_block ~block in let%map () = - match snark_work_opt with - | None -> - Deferred.Result.return () - | Some snark_work_str -> - let%bind Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } = - Deferred.return @@ decode_snark_work snark_work_str - in - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter - in - verify_snark_work ~proof ~message + if no_checks then return () + else + match snark_work_opt with + | None -> + Deferred.Result.return () + | Some snark_work_str -> + let%bind Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } = + Deferred.return @@ decode_snark_work snark_work_str + in + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter + in + verify_snark_work ~proof ~message in ( External_transition.state_hash block + , External_transition.parent block , External_transition.blockchain_length block , External_transition.global_slot block ) type valid_payload = { state_hash : State_hash.t + ; parent : State_hash.t ; height : Unsigned.uint32 ; slot : Unsigned.uint32 } -let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = +let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t + = `Assoc [ ("state_hash", State_hash.to_yojson state_hash) + ; ("parent", State_hash.to_yojson parent) ; ("height", `Int (Unsigned.UInt32.to_int height)) ; ("slot", `Int (Unsigned.UInt32.to_int slot)) ] @@ -133,14 +139,22 @@ let command = flag "--block-dir" ~aliases:[ "-block-dir" ] ~doc:"the path to the directory containing blocks for the submission" (required Filename.arg_type) - and inputs = anon (sequence ("filename" %: Filename.arg_type)) in + and inputs = anon (sequence ("filename" %: Filename.arg_type)) + and no_checks = + flag "--no-checks" ~aliases:[ "-no-checks" ] + ~doc: + "disable all the checks, just extract the info from the submissions" + no_arg + in fun () -> let open Deferred.Let_syntax in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> - match%bind validate_submission ~block_dir ~metadata_path with - | Ok (state_hash, height, slot) -> - display { state_hash; height; slot } ; + match%bind + validate_submission ~block_dir ~metadata_path ~no_checks + with + | Ok (state_hash, parent, height, slot) -> + display { state_hash; parent; height; slot } ; Deferred.unit | Error `Path_is_invalid -> display_error "path for metadata is invalid" ; From 1313513dd6f142e6733c668fc36b2ea2e7ad8539 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 26 Jan 2022 22:37:03 +0800 Subject: [PATCH 015/111] fix typo --- src/app/delegation_verify/delegation_verify.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 4e2c8d78afae..795af92059a7 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -104,7 +104,7 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = verify_snark_work ~proof ~message in ( External_transition.state_hash block - , External_transition.parent block + , External_transition.parent_hash block , External_transition.blockchain_length block , External_transition.global_slot block ) From 9da50994318f710658d10e492a7216229fa7b800 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 26 Jan 2022 23:39:08 +0800 Subject: [PATCH 016/111] delay the creation of the verifier --- .../delegation_verify/delegation_verify.ml | 10 ++++---- src/app/delegation_verify/verifier.ml | 24 +++++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 795af92059a7..15fe9cb9128a 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -57,11 +57,10 @@ let decode_block str = let verify_block ~block = let open External_transition in + let verify_blockchain_snarks = force Verifier.verify_blockchain_snarks in let%map result = - Verifier.verify_blockchain_snarks - [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) - ~proof:(protocol_state_proof block) - ] + verify_blockchain_snarks + [ (protocol_state block, protocol_state_proof block) ] in if result then Ok () else Error `Invalid_proof @@ -74,7 +73,8 @@ let decode_snark_work str = Error `Fail_to_decode_snark_work let verify_snark_work ~proof ~message = - let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in + let verify_transaction_snarks = force Verifier.verify_transaction_snarks in + let%map result = verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work let validate_submission ~block_dir ~metadata_path ~no_checks = diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index aa95cebf85f0..7762e481aa44 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -14,10 +14,30 @@ module B = Blockchain_snark.Blockchain_snark_state.Make (struct let proof_level = Genesis_constants.Proof_level.compiled end) -let verify_blockchain_snarks (chains : Blockchain_snark.Blockchain.t list) = +let verify_blockchain_snarks_ (chains : Blockchain_snark.Blockchain.t list) = B.Proof.verify @@ List.map chains ~f:(fun snark -> ( Blockchain_snark.Blockchain.state snark , Blockchain_snark.Blockchain.proof snark )) -let verify_transaction_snarks ts = T.verify ts +let verify_blockchain_snarks = + lazy + (let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = + Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled + end) in + B.Proof.verify) + +let verify_transaction_snarks = + lazy + (let module T = Transaction_snark.Make (struct + let constraint_constants = + Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled + end) in + T.verify) From f65a4719ecd7ec4af02fd288686bbd5f5d8de2df Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 27 Jan 2022 00:04:23 +0800 Subject: [PATCH 017/111] wrap verification in a pair of functions --- .../delegation_verify/delegation_verify.ml | 4 +- src/app/delegation_verify/verifier.ml | 43 ++++--------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 15fe9cb9128a..8518997fd61d 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -57,7 +57,7 @@ let decode_block str = let verify_block ~block = let open External_transition in - let verify_blockchain_snarks = force Verifier.verify_blockchain_snarks in + let verify_blockchain_snarks, _ = force Verifier.verify_functions in let%map result = verify_blockchain_snarks [ (protocol_state block, protocol_state_proof block) ] @@ -73,7 +73,7 @@ let decode_snark_work str = Error `Fail_to_decode_snark_work let verify_snark_work ~proof ~message = - let verify_transaction_snarks = force Verifier.verify_transaction_snarks in + let _, verify_transaction_snarks = force Verifier.verify_functions in let%map result = verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index 7762e481aa44..852ec044d983 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -1,43 +1,16 @@ -open Core - -module T = Transaction_snark.Make (struct - let constraint_constants = Genesis_constants.Constraint_constants.compiled - - let proof_level = Genesis_constants.Proof_level.compiled -end) - -module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - - let constraint_constants = Genesis_constants.Constraint_constants.compiled - - let proof_level = Genesis_constants.Proof_level.compiled -end) - -let verify_blockchain_snarks_ (chains : Blockchain_snark.Blockchain.t list) = - B.Proof.verify - @@ List.map chains ~f:(fun snark -> - ( Blockchain_snark.Blockchain.state snark - , Blockchain_snark.Blockchain.proof snark )) - -let verify_blockchain_snarks = +let verify_functions = lazy - (let module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - + (let module T = Transaction_snark.Make (struct let constraint_constants = Genesis_constants.Constraint_constants.compiled let proof_level = Genesis_constants.Proof_level.compiled end) in - B.Proof.verify) + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag -let verify_transaction_snarks = - lazy - (let module T = Transaction_snark.Make (struct - let constraint_constants = - Genesis_constants.Constraint_constants.compiled + let constraint_constants = Genesis_constants.Constraint_constants.compiled - let proof_level = Genesis_constants.Proof_level.compiled - end) in - T.verify) + let proof_level = Genesis_constants.Proof_level.compiled + end) in + (B.Proof.verify, T.verify)) From 2bdc2e03e5d3983fb16bc9281021e0af45f46d6c Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 27 Jan 2022 00:29:58 +0800 Subject: [PATCH 018/111] remove rpc_parallel stuff --- src/app/delegation_verify/delegation_verify.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 8518997fd61d..bd80db06c55e 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -179,4 +179,4 @@ let command = display_error "fail to verify the snark work" ; Deferred.unit)) -let () = Rpc_parallel.start_app command +let () = Async.Command.run command From 5238f496ac61f4cd45160a06da034b8001107377 Mon Sep 17 00:00:00 2001 From: georgeee Date: Fri, 8 Sep 2023 20:39:41 +0200 Subject: [PATCH 019/111] Fix after rebase onto latest compatible --- .../Dockerfile-delegation-stateless-verifier | 2 +- .../delegation_verify/delegation_verify.ml | 67 ++++++++++--------- src/app/delegation_verify/dune | 41 +++++++++++- src/app/delegation_verify/verifier.ml | 2 +- 4 files changed, 75 insertions(+), 37 deletions(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 8a6adc76f6e5..e607dc6b0efe 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -3,7 +3,7 @@ # - builds stateless verification tool # - should not include any data related to joining a specific network, only the node software itself ################################################################################################# -FROM gcr.io/o1labs-192920/mina-toolchain@sha256:7ef5827fa0854a0bcfdee69dcc0c2c7aef86e1662c630e71f07c1f9162e757fa AS builder +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:73562fcc35dcabd342f66f1d69ae12704e92d69edc0b37e7c88b4d11bc623f23 AS builder # Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI ARG DUNE_PROFILE=devnet diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index bd80db06c55e..6ca67887ae47 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -1,5 +1,4 @@ open Mina_base -open Mina_transition open Core open Async open Signature_lib @@ -30,39 +29,35 @@ let load_metadata str = (* This decoding is also unnecessarily complicated given that we are the creator of those data *) let decode_metadata str = - match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ok - { created_at = _ - ; peer_id = _ - ; snark_work - ; remote_addr = _ - ; submitter - ; block_hash - } -> ( - match submitter |> Public_key.Compressed.of_base58_check with - | Ok submitter -> - Ok (snark_work, submitter, block_hash) - | Error _ -> - Error `Fail_to_decode_metadata ) - | Error _ -> - Error `Fail_to_decode_metadata + let parsed_meta = + match Yojson.Safe.from_string str |> metadata_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error _ -> + Error `Fail_to_decode_metadata + in + let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in + let%bind.Result submitter = + Result.map_error ~f:(const `Fail_to_decode_metadata) + @@ Public_key.Compressed.of_base58_check submitter + in + Ok (snark_work, submitter, block_hash) let load_block ~block_dir ~block_hash = let block_path = Filename.concat block_dir (block_hash ^ ".dat") in try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block let decode_block str = - try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) with _ -> Error `Fail_to_decode_block let verify_block ~block = - let open External_transition in + let header = Mina_block.header block in + let open Mina_block.Header in let verify_blockchain_snarks, _ = force Verifier.verify_functions in - let%map result = - verify_blockchain_snarks - [ (protocol_state block, protocol_state_proof block) ] - in - if result then Ok () else Error `Invalid_proof + verify_blockchain_snarks + [ (protocol_state header, protocol_state_proof header) ] + |> Deferred.Result.map_error ~f:(const `Invalid_proof) let decode_snark_work str = match Base64.decode str with @@ -74,8 +69,8 @@ let decode_snark_work str = let verify_snark_work ~proof ~message = let _, verify_transaction_snarks = force Verifier.verify_functions in - let%map result = verify_transaction_snarks [ (proof, message) ] in - if result then Ok () else Error `Invalid_snark_work + verify_transaction_snarks [ (proof, message) ] + |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) let validate_submission ~block_dir ~metadata_path ~no_checks = let open Deferred.Result.Let_syntax in @@ -103,16 +98,22 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = in verify_snark_work ~proof ~message in - ( External_transition.state_hash block - , External_transition.parent_hash block - , External_transition.blockchain_length block - , External_transition.global_slot block ) + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) type valid_payload = { state_hash : State_hash.t ; parent : State_hash.t ; height : Unsigned.uint32 - ; slot : Unsigned.uint32 + ; slot : Mina_numbers.Global_slot_since_genesis.t } let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t @@ -121,7 +122,7 @@ let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t [ ("state_hash", State_hash.to_yojson state_hash) ; ("parent", State_hash.to_yojson parent) ; ("height", `Int (Unsigned.UInt32.to_int height)) - ; ("slot", `Int (Unsigned.UInt32.to_int slot)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int slot)) ] let display valid_payload = @@ -177,6 +178,6 @@ let command = Deferred.unit | Error `Invalid_snark_work -> display_error "fail to verify the snark work" ; - Deferred.unit)) + Deferred.unit )) let () = Async.Command.run command diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 7df3a9b2c5dd..1e7bc90927bc 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,5 +1,42 @@ (executable (name delegation_verify) - (libraries uptime_service core_kernel async mina_transition parallel) + (libraries + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base_import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_backend.pasta + kimchi_backend.pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + ) (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) + (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index 852ec044d983..fd8c5d66919f 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -13,4 +13,4 @@ let verify_functions = let proof_level = Genesis_constants.Proof_level.compiled end) in - (B.Proof.verify, T.verify)) + (B.Proof.verify, T.verify) ) From 0e8822a18c7834cf905d1cf5108beb821383d394 Mon Sep 17 00:00:00 2001 From: georgeee Date: Fri, 8 Sep 2023 23:10:38 +0200 Subject: [PATCH 020/111] Support graphql_control_port --- src/app/delegation_verify/delegation_verify.ml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6ca67887ae47..76dc7a61ab14 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,6 +10,7 @@ type metadata = ; remote_addr : string ; submitter : string ; block_hash : string + ; graphql_control_port: int option [@default None] } [@@deriving yojson] @@ -33,12 +34,12 @@ let decode_metadata str = match Yojson.Safe.from_string str |> metadata_of_yojson with | Ppx_deriving_yojson_runtime.Result.Ok a -> Ok a - | Ppx_deriving_yojson_runtime.Result.Error _ -> - Error `Fail_to_decode_metadata + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (`Fail_to_decode_metadata e) in let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in let%bind.Result submitter = - Result.map_error ~f:(const `Fail_to_decode_metadata) + Result.map_error ~f:(fun e -> `Fail_to_decode_metadata (Error.to_string_hum e)) @@ Public_key.Compressed.of_base58_check submitter in Ok (snark_work, submitter, block_hash) @@ -160,9 +161,12 @@ let command = | Error `Path_is_invalid -> display_error "path for metadata is invalid" ; exit 1 - | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> + | Error `Fail_to_load_metadata -> display_error "fail to load metadata" ; exit 2 + | Error (`Fail_to_decode_metadata e) -> + display_error ("fail to decode metadata: " ^ e); + exit 2 | Error `Fail_to_load_block -> display_error "fail to load block" ; exit 3 From bdc34eaa8ec8189d729988cf296689d737ebdf12 Mon Sep 17 00:00:00 2001 From: georgeee Date: Sun, 10 Sep 2023 15:37:30 +0200 Subject: [PATCH 021/111] Add --config-file optional argument Problem: delegation stateless verification tool is not usable for submissions on cluster with custum constraints constants. Solution: provide an option to load config file with custom constraints constants. Bonus: code prettified and reformated. --- .../delegation_verify/delegation_verify.ml | 94 ++++++++++++++----- src/app/delegation_verify/dune | 4 + src/app/delegation_verify/verifier.ml | 24 +++-- 3 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 76dc7a61ab14..b7c6650f1202 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,7 +10,7 @@ type metadata = ; remote_addr : string ; submitter : string ; block_hash : string - ; graphql_control_port: int option [@default None] + ; graphql_control_port : int option [@default None] } [@@deriving yojson] @@ -22,9 +22,6 @@ let get_filenames = | filenames -> filenames -(* This check seems unnecessary if the submission data is published by ourselfes *) -let check_path _ = Ok () - let load_metadata str = try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata @@ -39,7 +36,8 @@ let decode_metadata str = in let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in let%bind.Result submitter = - Result.map_error ~f:(fun e -> `Fail_to_decode_metadata (Error.to_string_hum e)) + Result.map_error ~f:(fun e -> + `Fail_to_decode_metadata (Error.to_string_hum e) ) @@ Public_key.Compressed.of_base58_check submitter in Ok (snark_work, submitter, block_hash) @@ -52,10 +50,9 @@ let decode_block str = try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) with _ -> Error `Fail_to_decode_block -let verify_block ~block = +let verify_block ~verify_blockchain_snarks ~block = let header = Mina_block.header block in let open Mina_block.Header in - let verify_blockchain_snarks, _ = force Verifier.verify_functions in verify_blockchain_snarks [ (protocol_state header, protocol_state_proof header) ] |> Deferred.Result.map_error ~f:(const `Invalid_proof) @@ -68,21 +65,23 @@ let decode_snark_work str = | Error _ -> Error `Fail_to_decode_snark_work -let verify_snark_work ~proof ~message = - let _, verify_transaction_snarks = force Verifier.verify_functions in +let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -let validate_submission ~block_dir ~metadata_path ~no_checks = +let validate_submission ~block_dir ~metadata_path ~no_checks + (verify_blockchain_snarks, verify_transaction_snarks) = let open Deferred.Result.Let_syntax in - let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in let%bind snark_work_opt, submitter, block_hash = Deferred.return @@ decode_metadata metadata_str in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = if no_checks then return () else verify_block ~block in + let%bind () = + if no_checks then return () + else verify_block ~verify_blockchain_snarks ~block + in let%map () = if no_checks then return () else @@ -97,7 +96,7 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = let message = Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in - verify_snark_work ~proof ~message + verify_snark_work ~verify_transaction_snarks ~proof ~message in let header = Mina_block.header block in let protocol_state = Mina_block.Header.protocol_state header in @@ -133,27 +132,72 @@ let display valid_payload = let display_error e = eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] +let config_flag = + let open Command.Param in + flag "--config-file" ~doc:"FILE config file" (optional string) + +let no_checks_flag = + let open Command.Param in + flag "--no-checks" ~aliases:[ "-no-checks" ] + ~doc:"disable all the checks, just extract the info from the submissions" + no_arg + +let block_dir_flag = + let open Command.Param in + flag "--block-dir" ~aliases:[ "-block-dir" ] + ~doc:"the path to the directory containing blocks for the submission" + (required Filename.arg_type) + +let instantiate_verify_functions ~logger = function + | None -> + Deferred.return + (Verifier.verify_functions + ~constraint_constants:Genesis_constants.Constraint_constants.compiled + ~proof_level:Genesis_constants.Proof_level.compiled () ) + | Some config_file -> + let%bind.Deferred precomputed_values = + let%bind.Deferred.Or_error config_json = + Genesis_ledger_helper.load_config_json config_file + in + let%bind.Deferred.Or_error config = + Deferred.return + @@ Result.map_error ~f:Error.of_string + @@ Runtime_config.of_yojson config_json + in + Genesis_ledger_helper.init_from_config_file ~logger ~proof_level:None + config + in + let%map.Deferred precomputed_values = + match precomputed_values with + | Ok (precomputed_values, _) -> + Deferred.return precomputed_values + | Error _ -> + display_error "fail to read config file" ; + exit 4 + in + let constraint_constants = + Precomputed_values.constraint_constants precomputed_values + in + Verifier.verify_functions ~constraint_constants ~proof_level:Full () + let command = Command.async ~summary:"A tool for verifying JSON payload submitted by the uptime service" Command.Let_syntax.( - let%map_open block_dir = - flag "--block-dir" ~aliases:[ "-block-dir" ] - ~doc:"the path to the directory containing blocks for the submission" - (required Filename.arg_type) + let%map_open block_dir = block_dir_flag and inputs = anon (sequence ("filename" %: Filename.arg_type)) - and no_checks = - flag "--no-checks" ~aliases:[ "-no-checks" ] - ~doc: - "disable all the checks, just extract the info from the submissions" - no_arg - in + and no_checks = no_checks_flag + and config_file = config_flag in fun () -> + let logger = Logger.create () in + let%bind.Deferred ver_funs = + instantiate_verify_functions ~logger config_file + in let open Deferred.Let_syntax in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> match%bind - validate_submission ~block_dir ~metadata_path ~no_checks + validate_submission ~block_dir ~metadata_path ~no_checks ver_funs with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; @@ -165,7 +209,7 @@ let command = display_error "fail to load metadata" ; exit 2 | Error (`Fail_to_decode_metadata e) -> - display_error ("fail to decode metadata: " ^ e); + display_error ("fail to decode metadata: " ^ e) ; exit 2 | Error `Fail_to_load_block -> display_error "fail to load block" ; diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 1e7bc90927bc..bf8815b7b31c 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -37,6 +37,10 @@ pickles pickles_types pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger ) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index fd8c5d66919f..43677f5d6405 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -1,16 +1,14 @@ -let verify_functions = - lazy - (let module T = Transaction_snark.Make (struct - let constraint_constants = - Genesis_constants.Constraint_constants.compiled +let verify_functions ~constraint_constants ~proof_level () = + let module T = Transaction_snark.Make (struct + let constraint_constants = constraint_constants - let proof_level = Genesis_constants.Proof_level.compiled - end) in - let module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag + let proof_level = proof_level + end) in + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag - let constraint_constants = Genesis_constants.Constraint_constants.compiled + let constraint_constants = constraint_constants - let proof_level = Genesis_constants.Proof_level.compiled - end) in - (B.Proof.verify, T.verify) ) + let proof_level = proof_level + end) in + (B.Proof.verify, T.verify) From ff8477d7fb23f0121ee24606fa92cb111706da18 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 9 Nov 2023 10:59:51 +0100 Subject: [PATCH 022/111] [delegation_verify] Refactor in order to abstract over input loading. --- .../delegation_verify/delegation_verify.ml | 207 ++++++++---------- src/app/delegation_verify/submission.ml | 89 ++++++++ 2 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 src/app/delegation_verify/submission.ml diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index b7c6650f1202..44121ca45f62 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -1,18 +1,19 @@ open Mina_base open Core open Async -open Signature_lib - -type metadata = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } -[@@deriving yojson] + +type error = + [ `Path_is_invalid + | `Fail_to_load_metadata + | `Fail_to_decode_metadata of string + | `Fail_to_load_block + | `Fail_to_decode_block + | `Invalid_proof + | `Fail_to_decode_snark_work + | `Invalid_snark_work + ] + +let unify_error e = (e :> error) let get_filenames = let open In_channel in @@ -22,34 +23,6 @@ let get_filenames = | filenames -> filenames -let load_metadata str = - try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata - -(* This decoding is also unnecessarily complicated given that we are the creator of those data *) -let decode_metadata str = - let parsed_meta = - match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a - | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (`Fail_to_decode_metadata e) - in - let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in - let%bind.Result submitter = - Result.map_error ~f:(fun e -> - `Fail_to_decode_metadata (Error.to_string_hum e) ) - @@ Public_key.Compressed.of_base58_check submitter - in - Ok (snark_work, submitter, block_hash) - -let load_block ~block_dir ~block_hash = - let block_path = Filename.concat block_dir (block_hash ^ ".dat") in - try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block - -let decode_block str = - try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) - with _ -> Error `Fail_to_decode_block - let verify_block ~verify_blockchain_snarks ~block = let header = Mina_block.header block in let open Mina_block.Header in @@ -57,57 +30,48 @@ let verify_block ~verify_blockchain_snarks ~block = [ (protocol_state header, protocol_state_proof header) ] |> Deferred.Result.map_error ~f:(const `Invalid_proof) -let decode_snark_work str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error `Fail_to_decode_snark_work ) - | Error _ -> - Error `Fail_to_decode_snark_work - let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -let validate_submission ~block_dir ~metadata_path ~no_checks - (verify_blockchain_snarks, verify_transaction_snarks) = - let open Deferred.Result.Let_syntax in - let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in - let%bind snark_work_opt, submitter, block_hash = - Deferred.return @@ decode_metadata metadata_str - in - let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in - let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = - if no_checks then return () - else verify_block ~verify_blockchain_snarks ~block - in - let%map () = - if no_checks then return () - else - match snark_work_opt with - | None -> - Deferred.Result.return () - | Some snark_work_str -> - let%bind Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } = - Deferred.return @@ decode_snark_work snark_work_str - in - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - in - let header = Mina_block.header block in - let protocol_state = Mina_block.Header.protocol_state header in - let consensus_state = - Mina_state.Protocol_state.consensus_state protocol_state - in - ( Mina_state.Protocol_state.hashes protocol_state - |> State_hash.State_hashes.state_hash - , Mina_state.Protocol_state.previous_state_hash protocol_state - , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) +module Validator(Sub : Submission.S) = struct + let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission = + let open Deferred.Result.Let_syntax in + let%bind block = + Sub.block submission + |> Result.map_error ~f:unify_error + |> Deferred.return + in + let%bind () = + if no_checks then return () + else verify_block ~verify_blockchain_snarks ~block + |> Deferred.Result.map_error ~f:unify_error + in + let%map () = + if no_checks then return () + else + match Sub.snark_work submission with + | None -> + Deferred.Result.return () + | Some Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:(Sub.submitter submission) + in + verify_snark_work ~verify_transaction_snarks ~proof ~message + |> Deferred.Result.map_error ~f:unify_error + in + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) +end type valid_payload = { state_hash : State_hash.t @@ -180,6 +144,33 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () +let handle_error = function + | `Path_is_invalid -> + display_error "path for metadata is invalid" ; + exit 1 + | `Fail_to_load_metadata -> + display_error "fail to load metadata" ; + exit 2 + | `Fail_to_decode_metadata e -> + display_error ("fail to decode metadata: " ^ e) ; + exit 2 + | `Fail_to_load_block -> + display_error "fail to load block" ; + exit 3 + | `Fail_to_decode_block -> + display_error "fail to decode block" ; + Deferred.unit + | `Invalid_proof -> + display_error + "fail to verify the protocol state proof inside the block" ; + Deferred.unit + | `Fail_to_decode_snark_work -> + display_error "fail to decode snark work" ; + Deferred.unit + | `Invalid_snark_work -> + display_error "fail to verify the snark work" ; + Deferred.unit + let command = Command.async ~summary:"A tool for verifying JSON payload submitted by the uptime service" @@ -190,42 +181,26 @@ let command = and config_file = config_flag in fun () -> let logger = Logger.create () in - let%bind.Deferred ver_funs = + let%bind.Deferred (verify_blockchain_snarks, verify_transaction_snarks) = instantiate_verify_functions ~logger config_file in let open Deferred.Let_syntax in - let metadata_pathes = get_filenames inputs in - Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> + let metadata_paths = get_filenames inputs in + let module V = Validator(Submission.JSON) in + Deferred.List.iter metadata_paths ~f:(fun path -> match%bind - validate_submission ~block_dir ~metadata_path ~no_checks ver_funs + ( let open Deferred.Result.Let_syntax in + let%bind submission = + Submission.JSON.load ~block_dir path + |> Result.map_error ~f:unify_error + |> Deferred.return + in + V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; Deferred.unit - | Error `Path_is_invalid -> - display_error "path for metadata is invalid" ; - exit 1 - | Error `Fail_to_load_metadata -> - display_error "fail to load metadata" ; - exit 2 - | Error (`Fail_to_decode_metadata e) -> - display_error ("fail to decode metadata: " ^ e) ; - exit 2 - | Error `Fail_to_load_block -> - display_error "fail to load block" ; - exit 3 - | Error `Fail_to_decode_block -> - display_error "fail to decode block" ; - Deferred.unit - | Error `Invalid_proof -> - display_error - "fail to verify the protocol state proof inside the block" ; - Deferred.unit - | Error `Fail_to_decode_snark_work -> - display_error "fail to decode snark work" ; - Deferred.unit - | Error `Invalid_snark_work -> - display_error "fail to verify the snark work" ; - Deferred.unit )) + | Error e -> + handle_error e)) let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml new file mode 100644 index 000000000000..5df847e4b0c3 --- /dev/null +++ b/src/app/delegation_verify/submission.ml @@ -0,0 +1,89 @@ +open Core +open Signature_lib + + type block_error = + [ `Fail_to_load_block + | `Fail_to_decode_block + ] + +module type S = sig + type t + + val snark_work : t -> Uptime_service.Proof_data.t option + + val submitter : t -> Public_key.Compressed.t + + val block_hash : t -> string + + val block : t -> (Mina_block.t, block_error) Result.t +end + + +module JSON = struct + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } + [@@deriving yojson] + + type t = + { snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + ; block_dir : string + } + + let snark_work { snark_work; _ } = snark_work + let submitter { submitter; _ } = submitter + let block_hash { block_hash; _ } = block_hash + + let block { block_dir; block_hash; _ } = + let open Result.Let_syntax in + let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in + let%bind contents = + try Ok (In_channel.read_all block_path) with + _ -> Error `Fail_to_load_block + in + try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) + with _ -> Error `Fail_to_decode_block + + let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error `Fail_to_decode_snark_work ) + | Error _ -> + Error `Fail_to_decode_snark_work + + let load ~block_dir filename = + let open Result.Let_syntax in + let%bind contents = + try Ok (In_channel.read_all filename) with _ -> Error `Fail_to_load_metadata + in + let%bind meta = + match Yojson.Safe.from_string contents |> raw_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (`Fail_to_decode_metadata e) + in + let%bind.Result submitter = + Result.map_error ~f:(fun e -> + `Fail_to_decode_metadata (Error.to_string_hum e) ) + @@ Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash; block_dir } +end From 68b9281c2056ea8b6b8b76f8ff12841397049d45 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 10 Nov 2023 16:38:50 +0100 Subject: [PATCH 023/111] [delegation_verify] Refactor error handling. --- .../delegation_verify/delegation_verify.ml | 83 +++++-------------- src/app/delegation_verify/submission.ml | 48 +++++------ 2 files changed, 41 insertions(+), 90 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 44121ca45f62..af505f286fa3 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -2,19 +2,6 @@ open Mina_base open Core open Async -type error = - [ `Path_is_invalid - | `Fail_to_load_metadata - | `Fail_to_decode_metadata of string - | `Fail_to_load_block - | `Fail_to_decode_block - | `Invalid_proof - | `Fail_to_decode_snark_work - | `Invalid_snark_work - ] - -let unify_error e = (e :> error) - let get_filenames = let open In_channel in function @@ -28,38 +15,33 @@ let verify_block ~verify_blockchain_snarks ~block = let open Mina_block.Header in verify_blockchain_snarks [ (protocol_state header, protocol_state_proof header) ] - |> Deferred.Result.map_error ~f:(const `Invalid_proof) let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] - |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -module Validator(Sub : Submission.S) = struct - let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission = +module Validator (Sub : Submission.S) = struct + let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks + submission = let open Deferred.Result.Let_syntax in - let%bind block = - Sub.block submission - |> Result.map_error ~f:unify_error - |> Deferred.return - in + let%bind block = Sub.block submission |> Deferred.return in let%bind () = if no_checks then return () else verify_block ~verify_blockchain_snarks ~block - |> Deferred.Result.map_error ~f:unify_error in let%map () = if no_checks then return () else match Sub.snark_work submission with | None -> - Deferred.Result.return () - | Some Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } -> - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:(Sub.submitter submission) - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - |> Deferred.Result.map_error ~f:unify_error + Deferred.Result.return () + | Some + Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } + -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:(Sub.submitter submission) + in + verify_snark_work ~verify_transaction_snarks ~proof ~message in let header = Mina_block.header block in let protocol_state = Mina_block.Header.protocol_state header in @@ -70,7 +52,8 @@ module Validator(Sub : Submission.S) = struct |> State_hash.State_hashes.state_hash , Mina_state.Protocol_state.previous_state_hash protocol_state , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state + ) end type valid_payload = @@ -144,36 +127,8 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () -let handle_error = function - | `Path_is_invalid -> - display_error "path for metadata is invalid" ; - exit 1 - | `Fail_to_load_metadata -> - display_error "fail to load metadata" ; - exit 2 - | `Fail_to_decode_metadata e -> - display_error ("fail to decode metadata: " ^ e) ; - exit 2 - | `Fail_to_load_block -> - display_error "fail to load block" ; - exit 3 - | `Fail_to_decode_block -> - display_error "fail to decode block" ; - Deferred.unit - | `Invalid_proof -> - display_error - "fail to verify the protocol state proof inside the block" ; - Deferred.unit - | `Fail_to_decode_snark_work -> - display_error "fail to decode snark work" ; - Deferred.unit - | `Invalid_snark_work -> - display_error "fail to verify the snark work" ; - Deferred.unit - -let command = - Command.async - ~summary:"A tool for verifying JSON payload submitted by the uptime service" +let filesystem_command = + Command.async ~summary:"Verify submissions and block read from the filesystem" Command.Let_syntax.( let%map_open block_dir = block_dir_flag and inputs = anon (sequence ("filename" %: Filename.arg_type)) @@ -192,7 +147,6 @@ let command = ( let open Deferred.Result.Let_syntax in let%bind submission = Submission.JSON.load ~block_dir path - |> Result.map_error ~f:unify_error |> Deferred.return in V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) @@ -201,6 +155,7 @@ let command = display { state_hash; parent; height; slot } ; Deferred.unit | Error e -> - handle_error e)) + display_error @@ Error.to_string_hum e ; + Deferred.unit)) let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5df847e4b0c3..09247d47f0e2 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -1,11 +1,6 @@ open Core open Signature_lib - type block_error = - [ `Fail_to_load_block - | `Fail_to_decode_block - ] - module type S = sig type t @@ -15,12 +10,10 @@ module type S = sig val block_hash : t -> string - val block : t -> (Mina_block.t, block_error) Result.t + val block : t -> Mina_block.t Or_error.t end - module JSON = struct - type raw = { created_at : string ; peer_id : string @@ -30,7 +23,7 @@ module JSON = struct ; block_hash : string ; graphql_control_port : int option [@default None] } - [@@deriving yojson] + [@@deriving yojson] type t = { snark_work : Uptime_service.Proof_data.t option @@ -40,50 +33,53 @@ module JSON = struct } let snark_work { snark_work; _ } = snark_work + let submitter { submitter; _ } = submitter + let block_hash { block_hash; _ } = block_hash let block { block_dir; block_hash; _ } = let open Result.Let_syntax in let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in let%bind contents = - try Ok (In_channel.read_all block_path) with - _ -> Error `Fail_to_load_block + try Ok (In_channel.read_all block_path) + with _ -> Error (Error.of_string "Fail to load block") in try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) - with _ -> Error `Fail_to_decode_block + with _ -> Error (Error.of_string "Fail to decode block") let decode_snark_work str = match Base64.decode str with | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error `Fail_to_decode_snark_work ) + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error (Error.of_string "Fail to decode snark work") ) | Error _ -> - Error `Fail_to_decode_snark_work + Error (Error.of_string "Fail to decode snark work") let load ~block_dir filename = let open Result.Let_syntax in let%bind contents = - try Ok (In_channel.read_all filename) with _ -> Error `Fail_to_load_metadata + try Ok (In_channel.read_all filename) + with _ -> Error (Error.of_string "Fail to load metadata") in let%bind meta = match Yojson.Safe.from_string contents |> raw_of_yojson with | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a + Ok a | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (`Fail_to_decode_metadata e) + Error (Error.of_string e) in let%bind.Result submitter = - Result.map_error ~f:(fun e -> - `Fail_to_decode_metadata (Error.to_string_hum e) ) - @@ Public_key.Compressed.of_base58_check meta.submitter + Public_key.Compressed.of_base58_check meta.submitter in let%map snark_work = - match meta.snark_work with - | None -> Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work in { submitter; snark_work; block_hash = meta.block_hash; block_dir } end + From b1cc46b8e8e68aa5588cb322a4197bc0caca5b8d Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 13 Nov 2023 14:42:49 +0100 Subject: [PATCH 024/111] Add an option to read data from Cassandra (unfinished). The setup depends on ~/.cassandra/cqlshrc configuration to work. --- src/app/delegation_verify/cassandra.ml | 81 +++++++++++++++++++ src/app/delegation_verify/cassandra.mli | 10 +++ .../delegation_verify/delegation_verify.ml | 61 +++++++++++--- src/app/delegation_verify/submission.ml | 1 - 4 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/app/delegation_verify/cassandra.ml create mode 100644 src/app/delegation_verify/cassandra.mli diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml new file mode 100644 index 000000000000..4acacae6a735 --- /dev/null +++ b/src/app/delegation_verify/cassandra.ml @@ -0,0 +1,81 @@ +open Async +open Core + +type connection = Process.t + +type 'a t = connection -> 'a Or_error.t Deferred.t + +module M = struct + type nonrec 'a t = 'a t + + let return x _ = return (Ok x) + + let map = `Custom (fun m ~f c -> m c >>| Or_error.map ~f) + + let bind m ~f c = + m c >>= function Error _ as e -> Deferred.return e | Ok x -> f x c +end + +include Monad.Make (M) + +let lift m _ = m + +(* Open connection to a Cassandra database using cqlsh executable + provided. The connection should be configured via ~/.cassandra/cqlshrc + configuration file. Return the process handle which can the be used to + send queries to the database. *) +let connect ?executable keyspace = + let open Deferred.Or_error.Let_syntax in + let prog = + Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const + |> Option.value ~default:"cqlsh" + in + let%map proc = Process.create ~prog ~args:[ "-k"; keyspace ] () in + Async.printf "Connected to the Cassandra database\n" ; + proc + +let exec ?cqlsh ~keyspace m = + let open Deferred.Or_error.Monad_infix in + connect ?executable:cqlsh keyspace >>= m + +let read_json_line chan = + let open Deferred.Let_syntax in + match%map Reader.read_line chan with + | `Eof -> + Or_error.error_string "Cassandra client: broken pipe" + | `Ok line -> ( + try + let j = Yojson.Safe.from_string line in + match Submission.JSON.raw_of_yojson j with + | Ppx_deriving_yojson_runtime.Result.Ok s -> + Ok s + | Ppx_deriving_yojson_runtime.Result.Error e -> + Or_error.error_string e + with Yojson.Json_error e -> Or_error.error_string e ) + +let rec read_json_output acc chan = + let open Deferred.Or_error.Let_syntax in + (* skip empty line *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + (* [json] header *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + (* [json] header *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with + | Ok s -> + read_json_output (s :: acc) chan + | Error _ -> + return (List.rev acc) + +let query q conn = + let open Deferred.Let_syntax in + let proc_stdin = Process.stdin conn in + Writer.write_line proc_stdin q ; + Writer.(write_line @@ Lazy.force stdout) q ; + let%bind () = Writer.close proc_stdin in + let%bind () = Writer.flushed proc_stdin in + match%bind read_json_output [] (Process.stdout conn) with + | Ok output -> + return (Ok output) + | Error e -> + return (Error e) diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli new file mode 100644 index 000000000000..c1e30f7e977a --- /dev/null +++ b/src/app/delegation_verify/cassandra.mli @@ -0,0 +1,10 @@ +open Async +open Core + +include Monad.S + +val lift : 'a Deferred.Or_error.t -> 'a t + +val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t + +val query : string -> Submission.JSON.raw list t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index af505f286fa3..8e41eab5c4aa 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -95,6 +95,17 @@ let block_dir_flag = ~doc:"the path to the directory containing blocks for the submission" (required Filename.arg_type) +let cassandra_executable_flag = + let open Command.Param in + flag "--executable" + ~aliases:[ "-executable"; "--cqlsh"; "-cqlsh" ] + ~doc:"the path to the cqlsh executable" + (optional Filename.arg_type) + +let timestamp = + let open Command.Param in + anon ("timestamp" %: string) + let instantiate_verify_functions ~logger = function | None -> Deferred.return @@ -136,26 +147,56 @@ let filesystem_command = and config_file = config_flag in fun () -> let logger = Logger.create () in - let%bind.Deferred (verify_blockchain_snarks, verify_transaction_snarks) = + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = instantiate_verify_functions ~logger config_file in let open Deferred.Let_syntax in let metadata_paths = get_filenames inputs in - let module V = Validator(Submission.JSON) in + let module V = Validator (Submission.JSON) in Deferred.List.iter metadata_paths ~f:(fun path -> match%bind - ( let open Deferred.Result.Let_syntax in - let%bind submission = - Submission.JSON.load ~block_dir path - |> Deferred.return - in - V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) + let open Deferred.Result.Let_syntax in + let%bind submission = + Submission.JSON.load ~block_dir path |> Deferred.return + in + V.validate ~no_checks ~verify_blockchain_snarks + ~verify_transaction_snarks submission with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; - Deferred.unit)) + display_error @@ Error.to_string_hum e ; + Deferred.unit )) + +let cassandra_command = + Command.async ~summary:"Verify submissions and block read from Cassandra" + Command.Let_syntax.( + let%map_open cqlsh = cassandra_executable_flag + and _period_start = timestamp + and _period_end = timestamp in + fun () -> + let open Deferred.Let_syntax in + match%bind + Cassandra.exec ?cqlsh ~keyspace:"bpu_integration_dev" + @@ Cassandra.query + "SELECT JSON created_at, peer_id, snark_work, remote_addr, \ + submitter, block_hash, graphql_control_port FROM submissions;" + with + | Ok subs -> + List.iter subs + ~f: + (Async.printf "submission: '%a'\n" (fun () s -> + Submission.JSON.raw_to_yojson s + |> Yojson.Safe.pretty_to_string ) ) ; + Deferred.unit + | Error e -> + display_error @@ Error.to_string_hum e ; + Deferred.unit) + +let command = + Command.group + ~summary:"A tool for verifying JSON payload submitted by the uptime service" + [ ("fs", filesystem_command); ("cassandra", cassandra_command) ] let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 09247d47f0e2..044220e642ca 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -82,4 +82,3 @@ module JSON = struct in { submitter; snark_work; block_hash = meta.block_hash; block_dir } end - From 510a61e66f419356265f7e88fc3f9754e1aab253 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 14 Nov 2023 12:24:36 +0100 Subject: [PATCH 025/111] [delegation_verify] Refactor to abstract over data source. --- src/app/delegation_verify/cassandra.ml | 15 +- src/app/delegation_verify/cassandra.mli | 2 +- .../delegation_verify/delegation_verify.ml | 140 ++++++++++-------- src/app/delegation_verify/dune | 2 + src/app/delegation_verify/known_blocks.ml | 67 +++++++++ src/app/delegation_verify/submission.ml | 130 ++++++++-------- 6 files changed, 225 insertions(+), 131 deletions(-) create mode 100644 src/app/delegation_verify/known_blocks.ml diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 4acacae6a735..f51a1bed3de4 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -46,21 +46,26 @@ let read_json_line chan = | `Ok line -> ( try let j = Yojson.Safe.from_string line in - match Submission.JSON.raw_of_yojson j with + match Submission.raw_of_yojson j with | Ppx_deriving_yojson_runtime.Result.Ok s -> Ok s | Ppx_deriving_yojson_runtime.Result.Error e -> Or_error.error_string e with Yojson.Json_error e -> Or_error.error_string e ) +let skip_line chan = + let open Deferred.Let_syntax in + let%map _ = Reader.read_line chan in + Ok () + let rec read_json_output acc chan = let open Deferred.Or_error.Let_syntax in (* skip empty line *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in - (* [json] header *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + let%bind () = skip_line chan in (* [json] header *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + let%bind () = skip_line chan in + (* separator line *) + let%bind () = skip_line chan in match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with | Ok s -> read_json_output (s :: acc) chan diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index c1e30f7e977a..8ae44c0d024e 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -7,4 +7,4 @@ val lift : 'a Deferred.Or_error.t -> 'a t val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t -val query : string -> Submission.JSON.raw list t +val query : string -> Submission.raw list t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 8e41eab5c4aa..e5dcf404105f 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,52 +10,9 @@ let get_filenames = | filenames -> filenames -let verify_block ~verify_blockchain_snarks ~block = - let header = Mina_block.header block in - let open Mina_block.Header in - verify_blockchain_snarks - [ (protocol_state header, protocol_state_proof header) ] - let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] -module Validator (Sub : Submission.S) = struct - let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks - submission = - let open Deferred.Result.Let_syntax in - let%bind block = Sub.block submission |> Deferred.return in - let%bind () = - if no_checks then return () - else verify_block ~verify_blockchain_snarks ~block - in - let%map () = - if no_checks then return () - else - match Sub.snark_work submission with - | None -> - Deferred.Result.return () - | Some - Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } - -> - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:(Sub.submitter submission) - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - in - let header = Mina_block.header block in - let protocol_state = Mina_block.Header.protocol_state header in - let consensus_state = - Mina_state.Protocol_state.consensus_state protocol_state - in - ( Mina_state.Protocol_state.hashes protocol_state - |> State_hash.State_hashes.state_hash - , Mina_state.Protocol_state.previous_state_hash protocol_state - , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state - ) -end - type valid_payload = { state_hash : State_hash.t ; parent : State_hash.t @@ -138,6 +95,66 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () +module Make_verifier (Source : Submission.Data_source) = struct + let verify_transaction_snarks = Source.verify_transaction_snarks + + let verify_blockchain_snarks = Source.verify_blockchain_snarks + + let intialize_submission ?validate (src : Source.t) (sub : Submission.t) = + if Known_blocks.is_known sub.block_hash then () + else + Known_blocks.add ?validate ~verify_blockchain_snarks + ~block_hash:sub.block_hash + (Source.load_block src ~block_hash:sub.block_hash) + + let verify ~validate (submission : Submission.t) = + let open Deferred.Result.Let_syntax in + let%bind block = Known_blocks.get submission.block_hash in + let%bind () = Known_blocks.is_valid submission.block_hash in + let%map () = + if validate then + match submission.snark_work with + | None -> + Deferred.Result.return () + | Some + Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } + -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:submission.submitter + in + verify_snark_work ~verify_transaction_snarks ~proof ~message + else return () + in + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state + ) + + let validate_and_display_results ~validate submission = + let open Deferred.Let_syntax in + match%map verify ~validate submission with + | Ok (state_hash, parent, height, slot) -> + display { state_hash; parent; height; slot } + | Error e -> + display_error @@ Error.to_string_hum e + + let process ?(validate = true) (src : Source.t) = + let open Deferred.Or_error.Let_syntax in + let%bind submissions = Source.load_submissions src in + List.iter submissions ~f:(intialize_submission ~validate src) ; + List.map submissions ~f:(validate_and_display_results ~validate) + |> Deferred.all_unit + |> Deferred.map ~f:Or_error.return +end + let filesystem_command = Command.async ~summary:"Verify submissions and block read from the filesystem" Command.Let_syntax.( @@ -150,24 +167,23 @@ let filesystem_command = let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = instantiate_verify_functions ~logger config_file in + let submission_paths = get_filenames inputs in + let module V = Make_verifier (struct + include Submission.Filesystem + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in let open Deferred.Let_syntax in - let metadata_paths = get_filenames inputs in - let module V = Validator (Submission.JSON) in - Deferred.List.iter metadata_paths ~f:(fun path -> - match%bind - let open Deferred.Result.Let_syntax in - let%bind submission = - Submission.JSON.load ~block_dir path |> Deferred.return - in - V.validate ~no_checks ~verify_blockchain_snarks - ~verify_transaction_snarks submission - with - | Ok (state_hash, parent, height, slot) -> - display { state_hash; parent; height; slot } ; - Deferred.unit - | Error e -> - display_error @@ Error.to_string_hum e ; - Deferred.unit )) + match%bind + V.process ~validate:(not no_checks) { submission_paths; block_dir } + with + | Ok () -> + Deferred.unit + | Error e -> + display_error @@ Error.to_string_hum e ; + exit 1) let cassandra_command = Command.async ~summary:"Verify submissions and block read from Cassandra" @@ -187,8 +203,8 @@ let cassandra_command = List.iter subs ~f: (Async.printf "submission: '%a'\n" (fun () s -> - Submission.JSON.raw_to_yojson s - |> Yojson.Safe.pretty_to_string ) ) ; + Submission.raw_to_yojson s |> Yojson.Safe.pretty_to_string ) + ) ; Deferred.unit | Error e -> display_error @@ Error.to_string_hum e ; diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index bf8815b7b31c..ad78d6b827d1 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -14,6 +14,8 @@ base64 integers async.async_command + sexplib0 + sexplib ; mina libs signature_lib mina_block diff --git a/src/app/delegation_verify/known_blocks.ml b/src/app/delegation_verify/known_blocks.ml new file mode 100644 index 000000000000..92913e964167 --- /dev/null +++ b/src/app/delegation_verify/known_blocks.ml @@ -0,0 +1,67 @@ +open Async +open Core + +module Block_hash = struct + open Sexplib.Conv + + type t = string [@@deriving sexp, compare] + + let hash = Base.String.hash +end + +module Deferred_block = struct + type t = + { block : Mina_block.t Deferred.Or_error.t + ; valid : unit Deferred.Or_error.t (* Raises if block is invalid. *) + } + + let block_of_string contents = + let compute v = + let r = + try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) + with _ -> Error (Error.of_string "Fail to decode block") + in + Async.Ivar.fill v r + in + Deferred.create compute + + let verify_block ~verify_blockchain_snarks block = + let header = Mina_block.header block in + let open Mina_block.Header in + verify_blockchain_snarks + [ (protocol_state header, protocol_state_proof header) ] + + let create ?(validate = true) ~verify_blockchain_snarks contents = + let open Deferred.Or_error.Monad_infix in + let block = contents >>= block_of_string in + let valid = + if validate then block >>= verify_block ~verify_blockchain_snarks + else Deferred.Or_error.return () + in + { block; valid } +end + +let known_blocks : (Block_hash.t, Deferred_block.t) Hashtbl.t = + Hashtbl.create (module Block_hash) + +let add ?validate ~verify_blockchain_snarks ~block_hash block_contents = + let block = + Deferred_block.create ?validate ~verify_blockchain_snarks block_contents + in + Hashtbl.add_exn known_blocks ~key:block_hash ~data:block + +let is_known hash = Hashtbl.mem known_blocks hash + +let get hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.block + +let is_valid hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.valid diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 044220e642ca..5fbd4c6a7a71 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -1,52 +1,45 @@ +open Async open Core open Signature_lib -module type S = sig - type t - - val snark_work : t -> Uptime_service.Proof_data.t option +type t = + { snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + } - val submitter : t -> Public_key.Compressed.t +type submission = t - val block_hash : t -> string - - val block : t -> Mina_block.t Or_error.t -end +type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } +[@@deriving yojson] -module JSON = struct - type raw = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } - [@@deriving yojson] +module type Data_source = sig + type t - type t = - { snark_work : Uptime_service.Proof_data.t option - ; submitter : Public_key.Compressed.t - ; block_hash : string - ; block_dir : string - } + val load_submissions : t -> submission list Deferred.Or_error.t - let snark_work { snark_work; _ } = snark_work + val load_block : block_hash:string -> t -> string Deferred.Or_error.t - let submitter { submitter; _ } = submitter + val verify_blockchain_snarks : + (Mina_wire_types.Mina_state_protocol_state.Value.V2.t * Mina_base.Proof.t) + list + -> unit Async_kernel__Deferred_or_error.t - let block_hash { block_hash; _ } = block_hash + val verify_transaction_snarks : + (Ledger_proof.t * Mina_base.Sok_message.t) list + -> (unit, Error.t) Deferred.Result.t +end - let block { block_dir; block_hash; _ } = - let open Result.Let_syntax in - let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in - let%bind contents = - try Ok (In_channel.read_all block_path) - with _ -> Error (Error.of_string "Fail to load block") - in - try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) - with _ -> Error (Error.of_string "Fail to decode block") +module Filesystem = struct + type t = { block_dir : string; submission_paths : string list } let decode_snark_work str = match Base64.decode str with @@ -56,29 +49,40 @@ module JSON = struct | Error _ -> Error (Error.of_string "Fail to decode snark work") - let load ~block_dir filename = - let open Result.Let_syntax in - let%bind contents = - try Ok (In_channel.read_all filename) - with _ -> Error (Error.of_string "Fail to load metadata") - in - let%bind meta = - match Yojson.Safe.from_string contents |> raw_of_yojson with - | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a - | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (Error.of_string e) - in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter; snark_work; block_hash = meta.block_hash; block_dir } + let load_submissions { submission_paths; _ } = + Deferred.create (fun ivar -> + List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> + let open Result.Let_syntax in + let%bind acc = acc in + let%bind contents = + try Ok (In_channel.read_all filename) + with _ -> Error (Error.of_string "Fail to load metadata") + in + let%bind meta = + match Yojson.Safe.from_string contents |> raw_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (Error.of_string e) + in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash } :: acc ) + |> Ivar.fill ivar ) + + let load_block ~block_hash { block_dir; _ } = + Deferred.create (fun ivar -> + let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in + ( try Ok (In_channel.read_all block_path) + with _ -> Error (Error.of_string "Fail to load block") ) + |> Ivar.fill ivar ) end From 4faba0f7080a2d4b995709ad5ce8b68f33046ee1 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 15 Nov 2023 14:09:17 +0100 Subject: [PATCH 026/111] [delegation_verify] Load input from Cassandra. --- src/app/delegation_verify/cassandra.ml | 102 +++++------------- src/app/delegation_verify/cassandra.mli | 17 +-- .../delegation_verify/delegation_verify.ml | 41 ++++--- src/app/delegation_verify/dune | 1 + src/app/delegation_verify/submission.ml | 99 +++++++++++++---- 5 files changed, 142 insertions(+), 118 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index f51a1bed3de4..dcd57762688f 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -1,86 +1,34 @@ open Async open Core -type connection = Process.t +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -type 'a t = connection -> 'a Or_error.t Deferred.t - -module M = struct - type nonrec 'a t = 'a t - - let return x _ = return (Ok x) - - let map = `Custom (fun m ~f c -> m c >>| Or_error.map ~f) - - let bind m ~f c = - m c >>= function Error _ as e -> Deferred.return e | Ok x -> f x c -end - -include Monad.Make (M) - -let lift m _ = m - -(* Open connection to a Cassandra database using cqlsh executable - provided. The connection should be configured via ~/.cassandra/cqlshrc - configuration file. Return the process handle which can the be used to - send queries to the database. *) -let connect ?executable keyspace = +let query ?executable ~parse q = let open Deferred.Or_error.Let_syntax in let prog = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in - let%map proc = Process.create ~prog ~args:[ "-k"; keyspace ] () in - Async.printf "Connected to the Cassandra database\n" ; - proc - -let exec ?cqlsh ~keyspace m = - let open Deferred.Or_error.Monad_infix in - connect ?executable:cqlsh keyspace >>= m - -let read_json_line chan = - let open Deferred.Let_syntax in - match%map Reader.read_line chan with - | `Eof -> - Or_error.error_string "Cassandra client: broken pipe" - | `Ok line -> ( - try - let j = Yojson.Safe.from_string line in - match Submission.raw_of_yojson j with - | Ppx_deriving_yojson_runtime.Result.Ok s -> - Ok s - | Ppx_deriving_yojson_runtime.Result.Error e -> - Or_error.error_string e - with Yojson.Json_error e -> Or_error.error_string e ) - -let skip_line chan = - let open Deferred.Let_syntax in - let%map _ = Reader.read_line chan in - Ok () - -let rec read_json_output acc chan = - let open Deferred.Or_error.Let_syntax in - (* skip empty line *) - let%bind () = skip_line chan in - (* [json] header *) - let%bind () = skip_line chan in - (* separator line *) - let%bind () = skip_line chan in - match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with - | Ok s -> - read_json_output (s :: acc) chan - | Error _ -> - return (List.rev acc) - -let query q conn = - let open Deferred.Let_syntax in - let proc_stdin = Process.stdin conn in - Writer.write_line proc_stdin q ; - Writer.(write_line @@ Lazy.force stdout) q ; - let%bind () = Writer.close proc_stdin in - let%bind () = Writer.flushed proc_stdin in - match%bind read_json_output [] (Process.stdout conn) with - | Ok output -> - return (Ok output) - | Error e -> - return (Error e) + printf "SQL: '%s'\n" q ; + let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in + List.slice data 3 (-2) (* skip header and footer *) + |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> + let open Or_error.Let_syntax in + let%bind l = acc in + try + let j = Yojson.Safe.from_string line in + match parse j with + | Ppx_deriving_yojson_runtime.Result.Ok s -> + Ok (s :: l) + | Ppx_deriving_yojson_runtime.Result.Error e -> + Or_error.error_string e + with Yojson.Json_error e -> Or_error.error_string e ) + |> Deferred.return + +let select ?executable ~keyspace ~parse ~fields ?where from = + query ?executable ~parse + @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" + (String.concat ~sep:"," fields) + keyspace + from + (match where with None -> "" | Some w -> " WHERE " ^ w) diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index 8ae44c0d024e..0a3a67337c7c 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -1,10 +1,15 @@ open Async -open Core -include Monad.S +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -val lift : 'a Deferred.Or_error.t -> 'a t +val query : + ?executable:string -> parse:'a parser -> string -> 'a list Deferred.Or_error.t -val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t - -val query : string -> Submission.raw list t +val select : + ?executable:string + -> keyspace:string + -> parse:'a parser + -> fields:string list + -> ?where:string + -> string + -> 'a list Deferred.Or_error.t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index e5dcf404105f..c5c42700fe04 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -189,26 +189,37 @@ let cassandra_command = Command.async ~summary:"Verify submissions and block read from Cassandra" Command.Let_syntax.( let%map_open cqlsh = cassandra_executable_flag - and _period_start = timestamp - and _period_end = timestamp in + and no_checks = no_checks_flag + and config_file = config_flag + and period_start = timestamp + and period_end = timestamp in fun () -> let open Deferred.Let_syntax in - match%bind - Cassandra.exec ?cqlsh ~keyspace:"bpu_integration_dev" - @@ Cassandra.query - "SELECT JSON created_at, peer_id, snark_work, remote_addr, \ - submitter, block_hash, graphql_control_port FROM submissions;" - with - | Ok subs -> - List.iter subs - ~f: - (Async.printf "submission: '%a'\n" (fun () s -> - Submission.raw_to_yojson s |> Yojson.Safe.pretty_to_string ) - ) ; + let logger = Logger.create () in + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = + instantiate_verify_functions ~logger config_file + in + let module V = Make_verifier (struct + include Submission.Cassandra + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in + let src = + Submission.Cassandra. + { executable = cqlsh + ; keyspace = "bpu_integration_dev" + ; period_start + ; period_end + } + in + match%bind V.process ~validate:(not no_checks) src with + | Ok () -> Deferred.unit | Error e -> display_error @@ Error.to_string_hum e ; - Deferred.unit) + exit 1) let command = Command.group diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index ad78d6b827d1..7c0f031cc18e 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -9,6 +9,7 @@ stdio base base.caml + hex ppx_deriving_yojson.runtime yojson base64 diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5fbd4c6a7a71..d860cb8a7a9c 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -21,6 +21,29 @@ type raw = } [@@deriving yojson] +let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error (Error.of_string "Fail to decode snark work") ) + | Error _ -> + Error (Error.of_string "Fail to decode snark work") + +let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash } + module type Data_source = sig type t @@ -41,14 +64,6 @@ end module Filesystem = struct type t = { block_dir : string; submission_paths : string list } - let decode_snark_work str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error (Error.of_string "Fail to decode snark work") ) - | Error _ -> - Error (Error.of_string "Fail to decode snark work") - let load_submissions { submission_paths; _ } = Deferred.create (fun ivar -> List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> @@ -65,18 +80,8 @@ module Filesystem = struct | Ppx_deriving_yojson_runtime.Result.Error e -> Error (Error.of_string e) in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter; snark_work; block_hash = meta.block_hash } :: acc ) + let%map t = of_raw meta in + t :: acc ) |> Ivar.fill ivar ) let load_block ~block_hash { block_dir; _ } = @@ -86,3 +91,57 @@ module Filesystem = struct with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) end + +module Cassandra = struct + type t = + { executable : string option + ; keyspace : string + ; period_start : string + ; period_end : string + } + + type block_data = { raw_block : string } [@@deriving yojson] + + let load_submissions { executable; keyspace; period_start; period_end } = + let open Deferred.Or_error.Let_syntax in + let%bind raw = + Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace + ~fields: + [ "created_at" + ; "peer_id" + ; "snark_work" + ; "remote_addr" + ; "submitter" + ; "block_hash" + ; "graphql_control_port" + ] + ~where: + (sprintf + "submitted_at_date = '%s' AND submitted_at >= '%s' AND \ + submitted_at <= '%s'" + (List.hd_exn @@ String.split ~on:' ' period_start) + period_start period_end ) + "submissions" + in + List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> + let open Result.Let_syntax in + let%bind l = acc in + printf "sub: '%s'\n" (Yojson.Safe.to_string @@ raw_to_yojson sub) ; + let%map s = of_raw sub in + s :: l ) + |> Deferred.return + + let load_block ~block_hash { executable; keyspace; _ } = + let open Deferred.Or_error.Let_syntax in + let%bind block_data = + Cassandra.select ?executable ~parse:block_data_of_yojson ~keyspace + ~fields:[ "raw_block" ] + ~where:(sprintf "block_hash = '%s'" block_hash) + "blocks" + in + match List.hd block_data with + | None -> + Deferred.Or_error.error_string "Cassandra: Block not found" + | Some b -> + Hex.Safe.of_hex b.raw_block |> Option.value_exn |> return +end From 1c5ecdd33bba963ea1a64e271556eaaad02344e6 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 10:36:13 +0100 Subject: [PATCH 027/111] Skip empty lines in order to avoid JSON parsing errors. An empty line sneaks in in particular when Casandra finds no results. --- src/app/delegation_verify/cassandra.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index dcd57762688f..b487bc06e07e 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -12,6 +12,7 @@ let query ?executable ~parse q = printf "SQL: '%s'\n" q ; let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in List.slice data 3 (-2) (* skip header and footer *) + |> List.filter ~f:(fun s -> not (String.is_empty s)) |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> let open Or_error.Let_syntax in let%bind l = acc in From bcffd44ea88f396643918a1b712056c9fa759da4 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 11:38:50 +0100 Subject: [PATCH 028/111] [delegation_verify] Fix problems with reading Cassandra blobs. --- src/app/delegation_verify/cassandra.ml | 3 +-- src/app/delegation_verify/dune | 2 +- src/app/delegation_verify/submission.ml | 11 ++++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index b487bc06e07e..5630f3a758a1 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -30,6 +30,5 @@ let select ?executable ~keyspace ~parse ~fields ?where from = query ?executable ~parse @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" (String.concat ~sep:"," fields) - keyspace - from + keyspace from (match where with None -> "" | Some w -> " WHERE " ^ w) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 7c0f031cc18e..3de6f7938743 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -9,7 +9,6 @@ stdio base base.caml - hex ppx_deriving_yojson.runtime yojson base64 @@ -17,6 +16,7 @@ async.async_command sexplib0 sexplib + hex ; mina libs signature_lib mina_block diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index d860cb8a7a9c..0e0dff8ae6d5 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -126,8 +126,12 @@ module Cassandra = struct List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> let open Result.Let_syntax in let%bind l = acc in - printf "sub: '%s'\n" (Yojson.Safe.to_string @@ raw_to_yojson sub) ; - let%map s = of_raw sub in + let snark_work = + Option.map sub.snark_work ~f:(fun s -> + String.chop_prefix_exn s ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> Base64.encode_string ) + in + let%map s = of_raw { sub with snark_work } in s :: l ) |> Deferred.return @@ -143,5 +147,6 @@ module Cassandra = struct | None -> Deferred.Or_error.error_string "Cassandra: Block not found" | Some b -> - Hex.Safe.of_hex b.raw_block |> Option.value_exn |> return + String.chop_prefix_exn b.raw_block ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> return end From 57c846efb969e7115949e6cabd897783b65fd139 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 14:04:27 +0100 Subject: [PATCH 029/111] [delegation_verify] Add identification data to the output. --- .../delegation_verify/delegation_verify.ml | 28 +++++++++++++------ src/app/delegation_verify/submission.ml | 9 ++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index c5c42700fe04..06f5f11b72b0 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -14,19 +14,22 @@ let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] type valid_payload = - { state_hash : State_hash.t + { created_at : string + ; submitter : string + ; state_hash : State_hash.t ; parent : State_hash.t ; height : Unsigned.uint32 ; slot : Mina_numbers.Global_slot_since_genesis.t } -let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t - = +let valid_payload_to_yojson (p : valid_payload) : Yojson.Safe.t = `Assoc - [ ("state_hash", State_hash.to_yojson state_hash) - ; ("parent", State_hash.to_yojson parent) - ; ("height", `Int (Unsigned.UInt32.to_int height)) - ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int slot)) + [ ("created_at", `String p.created_at) + ; ("submitter", `String p.submitter) + ; ("state_hash", State_hash.to_yojson p.state_hash) + ; ("parent", State_hash.to_yojson p.parent) + ; ("height", `Int (Unsigned.UInt32.to_int p.height)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) ] let display valid_payload = @@ -142,7 +145,16 @@ module Make_verifier (Source : Submission.Data_source) = struct let open Deferred.Let_syntax in match%map verify ~validate submission with | Ok (state_hash, parent, height, slot) -> - display { state_hash; parent; height; slot } + display + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + } | Error e -> display_error @@ Error.to_string_hum e diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 0e0dff8ae6d5..5baebda17822 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -3,7 +3,8 @@ open Core open Signature_lib type t = - { snark_work : Uptime_service.Proof_data.t option + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option ; submitter : Public_key.Compressed.t ; block_hash : string } @@ -42,7 +43,11 @@ let of_raw meta = let%map snark_work = decode_snark_work s in Some snark_work in - { submitter; snark_work; block_hash = meta.block_hash } + { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + } module type Data_source = sig type t From ca3bfde8b432c56f568abd4708a99b7c0c61163f Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 15:45:37 +0100 Subject: [PATCH 030/111] [delegation_verify] Abstract over output mechanism. In order to allow for outputting results to Cassandra. --- src/app/delegation_verify/cassandra.ml | 32 ++++++--- src/app/delegation_verify/cassandra.mli | 11 +++- .../delegation_verify/delegation_verify.ml | 66 ++++++------------- src/app/delegation_verify/output.ml | 28 ++++++++ src/app/delegation_verify/submission.ml | 13 ++++ 5 files changed, 91 insertions(+), 59 deletions(-) create mode 100644 src/app/delegation_verify/output.ml diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 5630f3a758a1..db6e2c1bfc7a 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -3,14 +3,23 @@ open Core type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -let query ?executable ~parse q = - let open Deferred.Or_error.Let_syntax in +let query ?executable q = let prog = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in printf "SQL: '%s'\n" q ; - let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in + Process.run_lines ~prog ~stdin:q ~args:[] () + +let select ?executable ~keyspace ~parse ~fields ?where from = + let open Deferred.Or_error.Let_syntax in + let%bind data = + query ?executable + @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" + (String.concat ~sep:"," fields) + keyspace from + (match where with None -> "" | Some w -> " WHERE " ^ w) + in List.slice data 3 (-2) (* skip header and footer *) |> List.filter ~f:(fun s -> not (String.is_empty s)) |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> @@ -26,9 +35,14 @@ let query ?executable ~parse q = with Yojson.Json_error e -> Or_error.error_string e ) |> Deferred.return -let select ?executable ~keyspace ~parse ~fields ?where from = - query ?executable ~parse - @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" - (String.concat ~sep:"," fields) - keyspace from - (match where with None -> "" | Some w -> " WHERE " ^ w) +let update ?executable ~keyspace ~table ~where updates = + let open Deferred.Or_error.Let_syntax in + let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in + let%map _ = + query ?executable + @@ Printf.sprintf "UPDATE %s.%s SET %s WHERE %s;" + keyspace table + (String.concat ~sep:"," assignments) + where + in + () diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index 0a3a67337c7c..990bd0de6b90 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -2,9 +2,6 @@ open Async type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -val query : - ?executable:string -> parse:'a parser -> string -> 'a list Deferred.Or_error.t - val select : ?executable:string -> keyspace:string @@ -13,3 +10,11 @@ val select : -> ?where:string -> string -> 'a list Deferred.Or_error.t + +val update : + ?executable:string + -> keyspace:string + -> table:string + -> where:string + -> (string * string) list + -> unit Deferred.Or_error.t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 06f5f11b72b0..3d534dd5980c 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -13,32 +13,6 @@ let get_filenames = let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] -type valid_payload = - { created_at : string - ; submitter : string - ; state_hash : State_hash.t - ; parent : State_hash.t - ; height : Unsigned.uint32 - ; slot : Mina_numbers.Global_slot_since_genesis.t - } - -let valid_payload_to_yojson (p : valid_payload) : Yojson.Safe.t = - `Assoc - [ ("created_at", `String p.created_at) - ; ("submitter", `String p.submitter) - ; ("state_hash", State_hash.to_yojson p.state_hash) - ; ("parent", State_hash.to_yojson p.parent) - ; ("height", `Int (Unsigned.UInt32.to_int p.height)) - ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) - ] - -let display valid_payload = - printf "%s\n" @@ Yojson.Safe.to_string - @@ valid_payload_to_yojson valid_payload - -let display_error e = - eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] - let config_flag = let open Command.Param in flag "--config-file" ~doc:"FILE config file" (optional string) @@ -90,7 +64,7 @@ let instantiate_verify_functions ~logger = function | Ok (precomputed_values, _) -> Deferred.return precomputed_values | Error _ -> - display_error "fail to read config file" ; + Output.display_error "fail to read config file" ; exit 4 in let constraint_constants = @@ -141,30 +115,28 @@ module Make_verifier (Source : Submission.Data_source) = struct , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) - let validate_and_display_results ~validate submission = + let validate_and_display_results ~validate ~src submission = let open Deferred.Let_syntax in - match%map verify ~validate submission with - | Ok (state_hash, parent, height, slot) -> - display - { created_at = submission.created_at - ; submitter = - Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter - ; state_hash - ; parent - ; height - ; slot - } - | Error e -> - display_error @@ Error.to_string_hum e + let%bind result = verify ~validate submission in + Result.map result ~f:(fun (state_hash, parent, height, slot) -> + Output. + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + }) + |> Source.output src submission let process ?(validate = true) (src : Source.t) = let open Deferred.Or_error.Let_syntax in let%bind submissions = Source.load_submissions src in List.iter submissions ~f:(intialize_submission ~validate src) ; - List.map submissions ~f:(validate_and_display_results ~validate) - |> Deferred.all_unit - |> Deferred.map ~f:Or_error.return + List.map submissions ~f:(validate_and_display_results ~src ~validate) + |> Deferred.Or_error.all_unit end let filesystem_command = @@ -194,7 +166,7 @@ let filesystem_command = | Ok () -> Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; + Output.display_error @@ Error.to_string_hum e ; exit 1) let cassandra_command = @@ -230,7 +202,7 @@ let cassandra_command = | Ok () -> Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; + Output.display_error @@ Error.to_string_hum e ; exit 1) let command = diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml new file mode 100644 index 000000000000..952f26140f84 --- /dev/null +++ b/src/app/delegation_verify/output.ml @@ -0,0 +1,28 @@ +open Async +open Mina_base + +type t = + { created_at : string + ; submitter : string + ; state_hash : State_hash.t + ; parent : State_hash.t + ; height : Unsigned.uint32 + ; slot : Mina_numbers.Global_slot_since_genesis.t + } + +let valid_payload_to_yojson (p : t) : Yojson.Safe.t = + `Assoc + [ ("created_at", `String p.created_at) + ; ("submitter", `String p.submitter) + ; ("state_hash", State_hash.to_yojson p.state_hash) + ; ("parent", State_hash.to_yojson p.parent) + ; ("height", `Int (Unsigned.UInt32.to_int p.height)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) + ] + +let display valid_payload = + printf "%s\n" @@ Yojson.Safe.to_string + @@ valid_payload_to_yojson valid_payload + +let display_error e = + eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5baebda17822..e6e8e9f9b063 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -64,6 +64,8 @@ module type Data_source = sig val verify_transaction_snarks : (Ledger_proof.t * Mina_base.Sok_message.t) list -> (unit, Error.t) Deferred.Result.t + + val output : t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t end module Filesystem = struct @@ -95,6 +97,15 @@ module Filesystem = struct ( try Ok (In_channel.read_all block_path) with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) + + let output _ (_submission : submission) = function + | Ok payload -> + Output.display payload ; + Deferred.Or_error.return () + + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Deferred.Or_error.return () end module Cassandra = struct @@ -154,4 +165,6 @@ module Cassandra = struct | Some b -> String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return + + let output = Filesystem.output end From 6322cb63ef80bb64a728a8a28b9705c4c3524167 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 16:25:14 +0100 Subject: [PATCH 031/111] [delegation_verify] Upload results to Cassandra. --- src/app/delegation_verify/cassandra.ml | 6 ++-- .../delegation_verify/delegation_verify.ml | 20 +++++------ src/app/delegation_verify/output.ml | 8 +++++ src/app/delegation_verify/submission.ml | 34 ++++++++++++++++--- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index db6e2c1bfc7a..56a2c378af2f 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -13,7 +13,7 @@ let query ?executable q = let select ?executable ~keyspace ~parse ~fields ?where from = let open Deferred.Or_error.Let_syntax in - let%bind data = + let%bind data = query ?executable @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" (String.concat ~sep:"," fields) @@ -38,9 +38,9 @@ let select ?executable ~keyspace ~parse ~fields ?where from = let update ?executable ~keyspace ~table ~where updates = let open Deferred.Or_error.Let_syntax in let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in - let%map _ = + let%map _ = query ?executable - @@ Printf.sprintf "UPDATE %s.%s SET %s WHERE %s;" + @@ Printf.sprintf "CONSISTENCY LOCAL_QUORUM; UPDATE %s.%s SET %s WHERE %s;" keyspace table (String.concat ~sep:"," assignments) where diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 3d534dd5980c..6a4cbfc423ca 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -119,16 +119,16 @@ module Make_verifier (Source : Submission.Data_source) = struct let open Deferred.Let_syntax in let%bind result = verify ~validate submission in Result.map result ~f:(fun (state_hash, parent, height, slot) -> - Output. - { created_at = submission.created_at - ; submitter = - Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter - ; state_hash - ; parent - ; height - ; slot - }) + Output. + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + } ) |> Source.output src submission let process ?(validate = true) (src : Source.t) = diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml index 952f26140f84..1d98d1dd55f4 100644 --- a/src/app/delegation_verify/output.ml +++ b/src/app/delegation_verify/output.ml @@ -20,6 +20,14 @@ let valid_payload_to_yojson (p : t) : Yojson.Safe.t = ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) ] +let valid_payload_to_cassandra_updates (p : t) = + [ ("height", Unsigned.UInt32.to_string p.height) + ; ("slot", Mina_numbers.Global_slot_since_genesis.to_string p.slot) + ; ("parent", Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.parent) + ; ( "state_hash" + , Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.state_hash ) + ] + let display valid_payload = printf "%s\n" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index e6e8e9f9b063..a3b34c523d84 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -65,7 +65,8 @@ module type Data_source = sig (Ledger_proof.t * Mina_base.Sok_message.t) list -> (unit, Error.t) Deferred.Result.t - val output : t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t + val output : + t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t end module Filesystem = struct @@ -100,9 +101,8 @@ module Filesystem = struct let output _ (_submission : submission) = function | Ok payload -> - Output.display payload ; - Deferred.Or_error.return () - + Output.display payload ; + Deferred.Or_error.return () | Error e -> Output.display_error @@ Error.to_string_hum e ; Deferred.Or_error.return () @@ -166,5 +166,29 @@ module Cassandra = struct String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return - let output = Filesystem.output + let output src (submission : submission) = function + | Ok payload -> + Output.display payload ; + Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.created_at) + submission.created_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + Output.(valid_payload_to_cassandra_updates payload) + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.created_at) + submission.created_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) ] end From 43f9a9d22d198db6c927d8f99437ef9b7a25bd67 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 17 Nov 2023 14:44:42 +0100 Subject: [PATCH 032/111] [delegation_verify] Use the proper keys for Cassandra records. --- .../delegation_verify/delegation_verify.ml | 26 +-- src/app/delegation_verify/submission.ml | 148 +++++++++++++----- 2 files changed, 123 insertions(+), 51 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6a4cbfc423ca..e7cbfffd8372 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -77,20 +77,22 @@ module Make_verifier (Source : Submission.Data_source) = struct let verify_blockchain_snarks = Source.verify_blockchain_snarks - let intialize_submission ?validate (src : Source.t) (sub : Submission.t) = - if Known_blocks.is_known sub.block_hash then () + let intialize_submission ?validate (src : Source.t) (sub : Source.submission) + = + let block_hash = Source.block_hash sub in + if Known_blocks.is_known block_hash then () else - Known_blocks.add ?validate ~verify_blockchain_snarks - ~block_hash:sub.block_hash - (Source.load_block src ~block_hash:sub.block_hash) + Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash + (Source.load_block src ~block_hash) - let verify ~validate (submission : Submission.t) = + let verify ~validate (submission : Source.submission) = let open Deferred.Result.Let_syntax in - let%bind block = Known_blocks.get submission.block_hash in - let%bind () = Known_blocks.is_valid submission.block_hash in + let block_hash = Source.block_hash submission in + let%bind block = Known_blocks.get block_hash in + let%bind () = Known_blocks.is_valid block_hash in let%map () = if validate then - match submission.snark_work with + match Source.snark_work submission with | None -> Deferred.Result.return () | Some @@ -98,7 +100,7 @@ module Make_verifier (Source : Submission.Data_source) = struct -> let message = Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:submission.submitter + ~prover:(Source.submitter submission) in verify_snark_work ~verify_transaction_snarks ~proof ~message else return () @@ -120,10 +122,10 @@ module Make_verifier (Source : Submission.Data_source) = struct let%bind result = verify ~validate submission in Result.map result ~f:(fun (state_hash, parent, height, slot) -> Output. - { created_at = submission.created_at + { created_at = Source.created_at submission ; submitter = Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter + (Source.submitter submission) ; state_hash ; parent ; height diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index a3b34c523d84..308bbca441ff 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -2,26 +2,6 @@ open Async open Core open Signature_lib -type t = - { created_at : string - ; snark_work : Uptime_service.Proof_data.t option - ; submitter : Public_key.Compressed.t - ; block_hash : string - } - -type submission = t - -type raw = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } -[@@deriving yojson] - let decode_snark_work str = match Base64.decode str with | Ok str -> ( @@ -30,28 +10,19 @@ let decode_snark_work str = | Error _ -> Error (Error.of_string "Fail to decode snark work") -let of_raw meta = - let open Result.Let_syntax in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter - ; snark_work - ; block_hash = meta.block_hash - ; created_at = meta.created_at - } - module type Data_source = sig type t + type submission + + val created_at : submission -> string + + val block_hash : submission -> string + + val snark_work : submission -> Uptime_service.Proof_data.t option + + val submitter : submission -> Public_key.Compressed.t + val load_submissions : t -> submission list Deferred.Or_error.t val load_block : block_hash:string -> t -> string Deferred.Or_error.t @@ -72,6 +43,51 @@ end module Filesystem = struct type t = { block_dir : string; submission_paths : string list } + type submission = + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + } + + let created_at ({ created_at; _ } : submission) = created_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + let load_submissions { submission_paths; _ } = Deferred.create (fun ivar -> List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> @@ -118,12 +134,66 @@ module Cassandra = struct type block_data = { raw_block : string } [@@deriving yojson] + type submission = + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + ; submitted_at : string + ; submitted_at_date : string + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + ; submitted_at : string + ; submitted_at_date : string + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + ( { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + ; submitted_at_date = meta.submitted_at_date + ; submitted_at = meta.submitted_at + } + : submission ) + + let created_at ({ created_at; _ } : submission) = created_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in let%bind raw = Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace ~fields: [ "created_at" + ; "submitted_at_date" + ; "submitted_at" ; "peer_id" ; "snark_work" ; "remote_addr" From 31ecc3fdf67c6f5c08279b438385c2a4da6e1fb3 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 17 Nov 2023 15:32:07 +0100 Subject: [PATCH 033/111] [delegation_verify] Allow for querying entries from multiple days. --- src/app/delegation_verify/submission.ml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 308bbca441ff..67b4162053cc 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -188,6 +188,20 @@ module Cassandra = struct let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in + let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in + let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in + let partition_keys = + Date.dates_between ~min:start_day ~max:end_day + |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") + in + let partition = + if List.length partition_keys = 1 then + sprintf "submitted_at_date = '%s'" (List.hd_exn partition_keys) + else + sprintf + "submitted_at_date IN (%s)" + (String.concat ~sep:"," @@ List.map ~f:(sprintf "'%s'") partition_keys) + in let%bind raw = Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace ~fields: @@ -203,9 +217,8 @@ module Cassandra = struct ] ~where: (sprintf - "submitted_at_date = '%s' AND submitted_at >= '%s' AND \ - submitted_at <= '%s'" - (List.hd_exn @@ String.split ~on:' ' period_start) + "%s AND submitted_at >= '%s' AND submitted_at <= '%s'" + partition period_start period_end ) "submissions" in From eb2ee341244bfeebc918f0cc405e13823cefc55c Mon Sep 17 00:00:00 2001 From: Smorci Date: Fri, 17 Nov 2023 17:14:53 +0000 Subject: [PATCH 034/111] PM-633 Add delegation_verify package and docker image, update libp2p hash --- flake.nix | 4 ++-- nix/docker.nix | 10 ++++++++++ nix/libp2p_helper.json | 2 +- nix/ocaml.nix | 24 +++++++++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 1bae798d9b02..0136412c2158 100644 --- a/flake.nix +++ b/flake.nix @@ -287,12 +287,12 @@ # Main user-facing binaries. packages = rec { inherit (ocamlPackages) - mina mina_tests mina-ocaml-format test_executive; + mina mina_tests mina-ocaml-format test_executive mina-delegation-verify; inherit (pkgs) libp2p_helper kimchi_bindings_stubs snarky_js leaderboard validation trace-tool zkapp-cli; inherit (dockerImages) - mina-image-slim mina-image-full mina-archive-image-full; + mina-image-slim mina-image-full mina-archive-image-full mina-delegation-verify-image; mina-deb = debianPackages.mina; default = mina; }; diff --git a/nix/docker.nix b/nix/docker.nix index 748c23369522..3655dc2fd37e 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -77,6 +77,16 @@ in { inherit created; contents = [ ocamlPackages_mina.mina.out ]; }; + + mina-delegation-verify-image = dockerTools.streamLayeredImage { + name = "mina-delegation-verify"; + inherit created; + contents = [ ocamlPackages_mina.mina-delegation-verify.out ]; + config = { + cmd = [ "/bin/delegation-verify" ]; + }; + }; + mina-image-full = mkFullImage "mina" (with ocamlPackages_mina; [ mina-build-config mina-daemon-scripts diff --git a/nix/libp2p_helper.json b/nix/libp2p_helper.json index 7a0a9b7e34a7..650889eddc1c 100644 --- a/nix/libp2p_helper.json +++ b/nix/libp2p_helper.json @@ -1 +1 @@ -{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} \ No newline at end of file +{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} diff --git a/nix/ocaml.nix b/nix/ocaml.nix index d6a0be0a9e88..defb546007a9 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -98,7 +98,8 @@ let --set MINA_LIBP2P_HELPER_PATH ${pkgs.libp2p_helper}/bin/libp2p_helper \ --set MINA_COMMIT_SHA1 ${escapeShellArg commit_sha1} \ --set MINA_COMMIT_DATE ${escapeShellArg commit_date} \ - --set MINA_BRANCH "''${MINA_BRANCH-}" + --set MINA_BRANCH "''${MINA_BRANCH-}" \ + --set TZDIR "${pkgs.tzdata}/share/zoneinfo" done '') package.outputs); @@ -321,6 +322,27 @@ let ''; }); + # Stateless verification tool + mina-delegation-verify-dev = self.mina-dev.overrideAttrs (oa: { + pname = "mina-delegation-verify"; + version = "dev"; + outputs = [ "out" ]; + + buildInputs = oa.buildInputs ++ [ pkgs.cassandra_4 ]; + + buildPhase = '' + dune build --display=short \ + src/app/delegation_verify/delegation_verify.exe + ''; + + installPhase = '' + mkdir -p $out/bin + mv _build/default/src/app/delegation_verify/delegation_verify.exe $out/bin/delegation-verify + ''; + }); + + mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; + # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { pname = "mina-test_executive"; From 88be126127c1350fa35bb35f2eaf6e95ab66c7a9 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 25 Oct 2021 11:11:21 +0800 Subject: [PATCH 035/111] add delegation verification tool --- .../delegation_verify/delegation_verify.ml | 192 ++++++++++++++++++ .../delegation_verify/delegation_verify.opam | 5 + src/app/delegation_verify/dune | 4 + 3 files changed, 201 insertions(+) create mode 100644 src/app/delegation_verify/delegation_verify.ml create mode 100644 src/app/delegation_verify/delegation_verify.opam create mode 100644 src/app/delegation_verify/dune diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml new file mode 100644 index 000000000000..6d5f9d4cc8f1 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.ml @@ -0,0 +1,192 @@ +open Mina_base +open Mina_transition +open Core +open Async +open Signature_lib + +type metadata = + { submitted_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + } +[@@deriving yojson] + +let get_filenames = + let open In_channel in + function + | [ "-" ] | [] -> + input_all stdin |> String.split_lines + | filenames -> + filenames + +(* This check seems unnecessary if the submission data is published by ourselfes *) +let check_path str = + match String.split str ~on:'/' with + | [ _basedir; submitter; block_hash; created_at ] -> ( + match submitter |> Yojson.Safe.from_string |> Public_key.of_yojson with + | Ok submitter -> ( + match Ptime.of_rfc3339 created_at with + | Ok (_, _, _) -> + Ok (submitter, block_hash) + | Error _ -> + Error `Path_is_invalid ) + | Error _ -> + Error `Path_is_invalid ) + | _ -> + Error `Path_is_invalid + +let load_metadata str = + try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata + +(* This decoding is also unnecessarily complicated given that we are the creator of those data *) +let decode_metadata str = + match Yojson.Safe.from_string str |> metadata_of_yojson with + | Ok { submitted_at = _; peer_id = _; snark_work; remote_addr = _ } -> + Ok snark_work + | Error _ -> + Error `Fail_to_decode_metadata + +let load_block ~block_dir ~block_hash = + let block_path = Filename.concat block_dir (block_hash ^ ".dat") in + try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block + +let decode_block str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + with _ -> Error `Fail_to_decode_block ) + | Error _ -> + Error `Fail_to_decode_block + +let verify_block ~verifier ~block = + let open External_transition in + match%map + Verifier.verify_blockchain_snarks verifier + [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) + ~proof:(protocol_state_proof block) + ] + with + | Ok result -> + if result then Ok () else Error `Invalid_proof + | Error e -> + Error (`Verifier_error e) + +let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error `Fail_to_decode_snark_work ) + | Error _ -> + Error `Fail_to_decode_snark_work + +let verify_snark_work ~verifier ~proof ~message = + match%map + Verifier.verify_transaction_snarks verifier [ (proof, message) ] + with + | Ok true -> + Ok () + | Ok false -> + Error `Invalid_snark_work + | Error e -> + Error (`Verifier_error e) + +let validate_submission ~verifier ~block_dir ~metadata_path = + let open Deferred.Result.Let_syntax in + let%bind submitter, block_hash = + Deferred.return @@ check_path metadata_path + in + let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in + let%bind snark_work_opt = Deferred.return @@ decode_metadata metadata_str in + let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in + let%bind block = Deferred.return @@ decode_block block_str in + let%bind () = verify_block ~verifier ~block in + let%map () = + match snark_work_opt with + | None -> + Deferred.Result.return () + | Some snark_work_str -> + let%bind Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } = + Deferred.return @@ decode_snark_work snark_work_str + in + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:(Public_key.compress submitter) + in + verify_snark_work ~verifier ~proof ~message + in + ( External_transition.state_hash block + , External_transition.blockchain_length block + , External_transition.global_slot block ) + +type valid_payload = + { state_hash : State_hash.t + ; height : Unsigned.uint32 + ; slot : Unsigned.uint32 + } + +let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = + `Assoc + [ ("state_hash", State_hash.to_yojson state_hash) + ; ("height", `Int (Unsigned.UInt32.to_int height)) + ; ("slot", `Int (Unsigned.UInt32.to_int slot)) + ] + +let display valid_payload = + printf "%s" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload + +let display_error e = + eprintf "%s" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] + +let command = + Command.async + ~summary:"A tool for verifying JSON payload submitted by the uptime service" + Command.Let_syntax.( + let%map_open block_dir = + flag "--block-dir" ~aliases:[ "-block-dir" ] + ~doc:"the path to the directory containing blocks for the submission" + (required Filename.arg_type) + and inputs = anon (sequence ("filename" %: Filename.arg_type)) in + fun () -> + let open Deferred.Let_syntax in + let logger = Logger.create () in + let%bind verifier = + Verifier.create ~logger + ~proof_level:Genesis_constants.Proof_level.compiled + ~constraint_constants: + Genesis_constants.Constraint_constants.compiled + ~pids:(Pid.Table.create ()) ~conf_dir:None + in + let metadata_pathes = get_filenames inputs in + Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> + match%bind + validate_submission ~verifier ~block_dir ~metadata_path + with + | Ok (state_hash, height, slot) -> + display { state_hash; height; slot } ; + Deferred.unit + | Error `Path_is_invalid -> + display_error "path for metadata is invalid" ; + exit 1 + | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> + display_error "fail to load metadata" ; + exit 2 + | Error `Fail_to_load_block | Error `Fail_to_decode_block -> + display_error "fail to load block" ; + exit 3 + | Error `Invalid_proof -> + display_error + "fail to verify the protocol state proof inside the block" ; + exit 5 + | Error (`Verifier_error e) -> + display_error @@ "verifier error: " ^ Error.to_string_hum e ; + exit 5 + | Error `Fail_to_decode_snark_work -> + display_error "fail to decode snark work" ; + exit 5 + | Error `Invalid_snark_work -> + display_error "fail to verify the snark work" ; + exit 5)) + +let () = Command.run command diff --git a/src/app/delegation_verify/delegation_verify.opam b/src/app/delegation_verify/delegation_verify.opam new file mode 100644 index 000000000000..7be19e3d6129 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.opam @@ -0,0 +1,5 @@ +opam-version: "2.0" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune new file mode 100644 index 000000000000..c683a012b03d --- /dev/null +++ b/src/app/delegation_verify/dune @@ -0,0 +1,4 @@ +(executable + (name delegation_verify) + (libraries uptime_service core_kernel async mina_transition) + (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) \ No newline at end of file From b472a26697b6920257f335711ce2e46c1da7faa0 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 25 Oct 2021 11:19:05 +0800 Subject: [PATCH 036/111] returns 0 even if the verification fails --- src/app/delegation_verify/delegation_verify.ml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6d5f9d4cc8f1..951030d0a8b2 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -178,15 +178,15 @@ let command = | Error `Invalid_proof -> display_error "fail to verify the protocol state proof inside the block" ; - exit 5 + Deferred.unit | Error (`Verifier_error e) -> display_error @@ "verifier error: " ^ Error.to_string_hum e ; - exit 5 + Deferred.unit | Error `Fail_to_decode_snark_work -> display_error "fail to decode snark work" ; - exit 5 + Deferred.unit | Error `Invalid_snark_work -> display_error "fail to verify the snark work" ; - exit 5)) + Deferred.unit)) let () = Command.run command From 9f50f2370ba1898e98e545388d3210b2cc5355ff Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 27 Oct 2021 03:23:25 +0800 Subject: [PATCH 037/111] adopt to new path structure for delegation data --- .../delegation_verify/delegation_verify.ml | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 951030d0a8b2..dcf84d84da1a 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -5,10 +5,12 @@ open Async open Signature_lib type metadata = - { submitted_at : string + { created_at : string ; peer_id : string ; snark_work : string option [@default None] ; remote_addr : string + ; submitter : string + ; block_hash : string } [@@deriving yojson] @@ -21,20 +23,7 @@ let get_filenames = filenames (* This check seems unnecessary if the submission data is published by ourselfes *) -let check_path str = - match String.split str ~on:'/' with - | [ _basedir; submitter; block_hash; created_at ] -> ( - match submitter |> Yojson.Safe.from_string |> Public_key.of_yojson with - | Ok submitter -> ( - match Ptime.of_rfc3339 created_at with - | Ok (_, _, _) -> - Ok (submitter, block_hash) - | Error _ -> - Error `Path_is_invalid ) - | Error _ -> - Error `Path_is_invalid ) - | _ -> - Error `Path_is_invalid +let check_path _ = Ok () let load_metadata str = try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata @@ -42,8 +31,19 @@ let load_metadata str = (* This decoding is also unnecessarily complicated given that we are the creator of those data *) let decode_metadata str = match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ok { submitted_at = _; peer_id = _; snark_work; remote_addr = _ } -> - Ok snark_work + | Ok + { created_at = _ + ; peer_id = _ + ; snark_work + ; remote_addr = _ + ; submitter + ; block_hash + } -> ( + match submitter |> Public_key.Compressed.of_base58_check with + | Ok submitter -> + Ok (snark_work, submitter, block_hash) + | Error _ -> + Error `Fail_to_decode_metadata ) | Error _ -> Error `Fail_to_decode_metadata @@ -52,12 +52,8 @@ let load_block ~block_dir ~block_hash = try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block let decode_block str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module External_transition.Stable.Latest) str) - with _ -> Error `Fail_to_decode_block ) - | Error _ -> - Error `Fail_to_decode_block + try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + with _ -> Error `Fail_to_decode_block let verify_block ~verifier ~block = let open External_transition in @@ -93,11 +89,11 @@ let verify_snark_work ~verifier ~proof ~message = let validate_submission ~verifier ~block_dir ~metadata_path = let open Deferred.Result.Let_syntax in - let%bind submitter, block_hash = - Deferred.return @@ check_path metadata_path - in + let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in - let%bind snark_work_opt = Deferred.return @@ decode_metadata metadata_str in + let%bind snark_work_opt, submitter, block_hash = + Deferred.return @@ decode_metadata metadata_str + in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in let%bind () = verify_block ~verifier ~block in @@ -111,8 +107,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = Deferred.return @@ decode_snark_work snark_work_str in let message = - Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:(Public_key.compress submitter) + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in verify_snark_work ~verifier ~proof ~message in @@ -156,7 +151,8 @@ let command = ~proof_level:Genesis_constants.Proof_level.compiled ~constraint_constants: Genesis_constants.Constraint_constants.compiled - ~pids:(Pid.Table.create ()) ~conf_dir:None + ~pids:(Child_processes.Termination.create_pid_table ()) + ~conf_dir:None in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> @@ -189,4 +185,4 @@ let command = display_error "fail to verify the snark work" ; Deferred.unit)) -let () = Command.run command +let () = Rpc_parallel.start_app command From 2e94b649e0211e06c490d76c4a613b5f3b501476 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 27 Oct 2021 03:37:33 +0800 Subject: [PATCH 038/111] disable logs --- src/app/delegation_verify/delegation_verify.ml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index dcf84d84da1a..357fbdc1fbea 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -129,10 +129,11 @@ let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = ] let display valid_payload = - printf "%s" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload + printf "%s\n" @@ Yojson.Safe.to_string + @@ valid_payload_to_yojson valid_payload let display_error e = - eprintf "%s" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] + eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] let command = Command.async @@ -145,7 +146,7 @@ let command = and inputs = anon (sequence ("filename" %: Filename.arg_type)) in fun () -> let open Deferred.Let_syntax in - let logger = Logger.create () in + let logger = Logger.null () in let%bind verifier = Verifier.create ~logger ~proof_level:Genesis_constants.Proof_level.compiled From 90fb0c5140821f3761592d1278656d9b1c4e5acb Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 01:30:50 +0800 Subject: [PATCH 039/111] adding dockerfiles for stateless verification tool --- dockerfiles/stages/4-stateless-verify | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 dockerfiles/stages/4-stateless-verify diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/stages/4-stateless-verify new file mode 100644 index 000000000000..bbefc57e2746 --- /dev/null +++ b/dockerfiles/stages/4-stateless-verify @@ -0,0 +1,54 @@ +################################################################################################# +# The "stateless verification build" Stage +# - builds stateless verification tool +# - should not include any data related to joining a specific network, only the node software itself +################################################################################################# +FROM opam-deps AS builder + +# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI +ARG DUNE_PROFILE=devnet + +# branch to checkout on first clone (this will be the only availible branch in the container) +# can also be a tagged release +ARG MINA_BRANCH=compatible + +# repo to checkout the branch from +ARG MINA_REPO=https://github.com/MinaProtocol/mina + +# location of repo used for pins and external package commits +ARG MINA_DIR=mina + +ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" + +# git will clone into an empty dir, but this also helps us set the workdir in advance +RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ + && git clone \ + -b "${MINA_BRANCH}" \ + --depth 1 \ + --shallow-submodules \ + --recurse-submodules \ + ${MINA_REPO} ${HOME}/${MINA_DIR} + +WORKDIR $HOME/${MINA_DIR} + +RUN mkdir ${HOME}/app + +# Build libp2p_helper +RUN make libp2p_helper \ + && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper + +# HACK: build without special cpu features to allow more people to run mina-rosetta +RUN ./scripts/zexe-standardize.sh + +# Make rosetta-crucial components and the generate_keypair tool +RUN eval $(opam config env) \ + && dune build --profile=${DUNE_PROFILE} \ + src/app/delegation_verify.exe \ + && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ + && rm -rf _build + +# Clear go module caches +RUN cd src/app/libp2p_helper/src/libp2p_helper \ + && go clean --cache --modcache --testcache -r + +ENTRYPOINT ["delegation-verify"] \ No newline at end of file From 69c8735fd74811929d50701ccd67a93c77c4751a Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 02:13:09 +0800 Subject: [PATCH 040/111] add delegation-verify service --- scripts/release-docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 1c69f91c0b89..cd4fbdd12df4 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -134,6 +134,9 @@ itn-orchestrator) DOCKER_CONTEXT="src/app/itn_orchestrator" ;; +delegation-verify) + DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-toolchain dockerfiles/stages/3-opam-deps dockerfiles/stages/4-stateless-verify" + ;; esac From 1dd68ded0ab0a0fd73172f5ac8189b19edab3e0b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 19:11:37 +0800 Subject: [PATCH 041/111] remote libp2p&go stuff in docker file --- dockerfiles/stages/4-stateless-verify | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/stages/4-stateless-verify index bbefc57e2746..b8a5e09b54c7 100644 --- a/dockerfiles/stages/4-stateless-verify +++ b/dockerfiles/stages/4-stateless-verify @@ -33,22 +33,13 @@ WORKDIR $HOME/${MINA_DIR} RUN mkdir ${HOME}/app -# Build libp2p_helper -RUN make libp2p_helper \ - && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper - -# HACK: build without special cpu features to allow more people to run mina-rosetta +# HACK: build without special cpu features to allow more people to run delegation verification tool RUN ./scripts/zexe-standardize.sh -# Make rosetta-crucial components and the generate_keypair tool RUN eval $(opam config env) \ && dune build --profile=${DUNE_PROFILE} \ src/app/delegation_verify.exe \ && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ && rm -rf _build -# Clear go module caches -RUN cd src/app/libp2p_helper/src/libp2p_helper \ - && go clean --cache --modcache --testcache -r - ENTRYPOINT ["delegation-verify"] \ No newline at end of file From c311632eddb569f214325832e77cbf1a0d0950f2 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 19:20:50 +0800 Subject: [PATCH 042/111] add readme --- src/app/delegation_verify/README.md | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/app/delegation_verify/README.md diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md new file mode 100644 index 000000000000..806ebcd85504 --- /dev/null +++ b/src/app/delegation_verify/README.md @@ -0,0 +1,58 @@ +This PR implements the stateless verification tool that would be used to verify that data collected by the uptime service. + +Usage: +``` +./delegation-verify --block-dir ... +./delegation-verify --block-dir - +``` +Where `` is a file path of the form `//-.json` +containing JSON following the format described in [Cloud storage section of delegation backend spec](https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage) (`` is a place holder for some directory): + +```json +{ "created_at": "server's timestamp (of the time of submission)" +, "remote_addr": "ip:port address from which request has come" +, "peer_id": "peer id (as in user's JSON submission)" +, "snark_work": "(optional) base64-encoded snark work blob" +, "block_hash": "base58check-encoded hash of a block" +, "submitter": "base58check-encoded submitter's public key" +} +``` + +File path content: +- `submitted_at_date` with server's date (of the time of submission) in format `YYYY-MM-DD` +- `submitted_at` with server's timestamp (of the time of submission) in RFC-3339 +- `submitter` is base58check-encoded submitter's public key + + +Parameter `--block-dir` specifies the path to the directory containing block for the submission. It is expected that blocks with `` exist under path `/.dat` for all of the filepaths provided. + +When `-` symbol is used as the `filename1`, no other `filename{i}` are accepted and all filenames will be read from the `stdin`, one filename per line. + +Utility `./delegation-verify` exits with the following exit codes: + +- `0` if it was able to execute verification of all filepaths + - Note that `0` is returned even when some of the payloads were marked as invalid +- `1` if one of `` files violated the naming convention +- `2` if the utility failed to read some of `` files +- `3` if the utility failed to read some of `/.dat` files +- `5` if the other error occurred + +In case of non-zero exit code, some details of the error may be printed to `stderr`. + +For each `` parameter one line of output is printed to `stdout` : + +- Valid payload at `` : + + ```json + { "state_hash": "" + , "height": "" + , "slot": "" + } + ``` + +- Invalid payload at `` : + + ```json + { "error": "" } + ``` +The stateless verification would check the protocol_state_proof in the block and the transaction_snark_proof in the snark_work. \ No newline at end of file From 20cbb4eccd6caac0a7c25838bc3c77c7b5e96b22 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 28 Oct 2021 21:13:19 +0800 Subject: [PATCH 043/111] add delegation-verify as valid service --- scripts/release-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index cd4fbdd12df4..44c75bce1523 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -10,7 +10,7 @@ set +x CLEAR='\033[0m' RED='\033[0;31m' # Array of valid service names -VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'itn-orchestrator') +VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'delegation-verify' ) function usage() { if [[ -n "$1" ]]; then From ab158c093d19b9cef225d1d302f119b78746767d Mon Sep 17 00:00:00 2001 From: georgeee Date: Thu, 28 Oct 2021 20:32:15 +0300 Subject: [PATCH 044/111] Use base image from URL --- ...ss-verify => Dockerfile-delegation-stateless-verifier} | 8 ++++---- scripts/release-docker.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename dockerfiles/{stages/4-stateless-verify => Dockerfile-delegation-stateless-verifier} (82%) diff --git a/dockerfiles/stages/4-stateless-verify b/dockerfiles/Dockerfile-delegation-stateless-verifier similarity index 82% rename from dockerfiles/stages/4-stateless-verify rename to dockerfiles/Dockerfile-delegation-stateless-verifier index b8a5e09b54c7..8a6adc76f6e5 100644 --- a/dockerfiles/stages/4-stateless-verify +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -3,7 +3,7 @@ # - builds stateless verification tool # - should not include any data related to joining a specific network, only the node software itself ################################################################################################# -FROM opam-deps AS builder +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:7ef5827fa0854a0bcfdee69dcc0c2c7aef86e1662c630e71f07c1f9162e757fa AS builder # Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI ARG DUNE_PROFILE=devnet @@ -38,8 +38,8 @@ RUN ./scripts/zexe-standardize.sh RUN eval $(opam config env) \ && dune build --profile=${DUNE_PROFILE} \ - src/app/delegation_verify.exe \ - && mv _build/default/src/app/delegation_verify/delegation_verify.exe /usr/local/bin/delegation-verify \ + src/app/delegation_verify/delegation_verify.exe \ + && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ && rm -rf _build -ENTRYPOINT ["delegation-verify"] \ No newline at end of file +ENTRYPOINT ["./delegation-verify"] diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 44c75bce1523..12db91c57283 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -135,7 +135,7 @@ itn-orchestrator) ;; delegation-verify) - DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-toolchain dockerfiles/stages/3-opam-deps dockerfiles/stages/4-stateless-verify" + DOCKERFILE_PATH="dockerfiles/Dockerfile-delegation-stateless-verifier" ;; esac From ece240948f08bc86b02fe8bec756361629f30a17 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 1 Nov 2021 13:13:08 +0800 Subject: [PATCH 045/111] inline verifier --- .../delegation_verify/delegation_verify.ml | 55 ++++++------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 357fbdc1fbea..0541bb85d6d4 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -55,18 +55,15 @@ let decode_block str = try Ok (Binable.of_string (module External_transition.Stable.Latest) str) with _ -> Error `Fail_to_decode_block -let verify_block ~verifier ~block = +let verify_block ~block = let open External_transition in - match%map - Verifier.verify_blockchain_snarks verifier + let%map result = + Verifier.verify_blockchain_snarks [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) ~proof:(protocol_state_proof block) ] - with - | Ok result -> - if result then Ok () else Error `Invalid_proof - | Error e -> - Error (`Verifier_error e) + in + if result then Ok () else Error `Invalid_proof let decode_snark_work str = match Base64.decode str with @@ -76,18 +73,11 @@ let decode_snark_work str = | Error _ -> Error `Fail_to_decode_snark_work -let verify_snark_work ~verifier ~proof ~message = - match%map - Verifier.verify_transaction_snarks verifier [ (proof, message) ] - with - | Ok true -> - Ok () - | Ok false -> - Error `Invalid_snark_work - | Error e -> - Error (`Verifier_error e) - -let validate_submission ~verifier ~block_dir ~metadata_path = +let verify_snark_work ~proof ~message = + let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in + if result then Ok () else Error `Invalid_snark_work + +let validate_submission ~block_dir ~metadata_path = let open Deferred.Result.Let_syntax in let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in @@ -96,7 +86,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = verify_block ~verifier ~block in + let%bind () = verify_block ~block in let%map () = match snark_work_opt with | None -> @@ -109,7 +99,7 @@ let validate_submission ~verifier ~block_dir ~metadata_path = let message = Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in - verify_snark_work ~verifier ~proof ~message + verify_snark_work ~proof ~message in ( External_transition.state_hash block , External_transition.blockchain_length block @@ -146,20 +136,9 @@ let command = and inputs = anon (sequence ("filename" %: Filename.arg_type)) in fun () -> let open Deferred.Let_syntax in - let logger = Logger.null () in - let%bind verifier = - Verifier.create ~logger - ~proof_level:Genesis_constants.Proof_level.compiled - ~constraint_constants: - Genesis_constants.Constraint_constants.compiled - ~pids:(Child_processes.Termination.create_pid_table ()) - ~conf_dir:None - in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> - match%bind - validate_submission ~verifier ~block_dir ~metadata_path - with + match%bind validate_submission ~block_dir ~metadata_path with | Ok (state_hash, height, slot) -> display { state_hash; height; slot } ; Deferred.unit @@ -169,16 +148,16 @@ let command = | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> display_error "fail to load metadata" ; exit 2 - | Error `Fail_to_load_block | Error `Fail_to_decode_block -> + | Error `Fail_to_load_block -> display_error "fail to load block" ; exit 3 + | Error `Fail_to_decode_block -> + display_error "fail to decode block" ; + Deferred.unit | Error `Invalid_proof -> display_error "fail to verify the protocol state proof inside the block" ; Deferred.unit - | Error (`Verifier_error e) -> - display_error @@ "verifier error: " ^ Error.to_string_hum e ; - Deferred.unit | Error `Fail_to_decode_snark_work -> display_error "fail to decode snark work" ; Deferred.unit From b046f4f3df3e5e5a7450526fe71bb0b519230b41 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 1 Nov 2021 13:14:07 +0800 Subject: [PATCH 046/111] inline verifier --- src/app/delegation_verify/verifier.ml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/app/delegation_verify/verifier.ml diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml new file mode 100644 index 000000000000..aa95cebf85f0 --- /dev/null +++ b/src/app/delegation_verify/verifier.ml @@ -0,0 +1,23 @@ +open Core + +module T = Transaction_snark.Make (struct + let constraint_constants = Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled +end) + +module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled +end) + +let verify_blockchain_snarks (chains : Blockchain_snark.Blockchain.t list) = + B.Proof.verify + @@ List.map chains ~f:(fun snark -> + ( Blockchain_snark.Blockchain.state snark + , Blockchain_snark.Blockchain.proof snark )) + +let verify_transaction_snarks ts = T.verify ts From efddba6c66d32526ffdcefbe0404cdf412dd816b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Tue, 2 Nov 2021 14:38:10 +0800 Subject: [PATCH 047/111] adding bisect_ppx --- src/app/delegation_verify/dune | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index c683a012b03d..7df3a9b2c5dd 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,4 +1,5 @@ (executable (name delegation_verify) - (libraries uptime_service core_kernel async mina_transition) - (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) \ No newline at end of file + (libraries uptime_service core_kernel async mina_transition parallel) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) From 9e8aba003781391c1b34269620c15943a4c74994 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Tue, 25 Jan 2022 19:44:14 +0800 Subject: [PATCH 048/111] adding an option "no-checks" to bypass all the checks --- .../delegation_verify/delegation_verify.ml | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 0541bb85d6d4..4e2c8d78afae 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -77,7 +77,7 @@ let verify_snark_work ~proof ~message = let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work -let validate_submission ~block_dir ~metadata_path = +let validate_submission ~block_dir ~metadata_path ~no_checks = let open Deferred.Result.Let_syntax in let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in @@ -86,34 +86,40 @@ let validate_submission ~block_dir ~metadata_path = in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = verify_block ~block in + let%bind () = if no_checks then return () else verify_block ~block in let%map () = - match snark_work_opt with - | None -> - Deferred.Result.return () - | Some snark_work_str -> - let%bind Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } = - Deferred.return @@ decode_snark_work snark_work_str - in - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter - in - verify_snark_work ~proof ~message + if no_checks then return () + else + match snark_work_opt with + | None -> + Deferred.Result.return () + | Some snark_work_str -> + let%bind Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } = + Deferred.return @@ decode_snark_work snark_work_str + in + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter + in + verify_snark_work ~proof ~message in ( External_transition.state_hash block + , External_transition.parent block , External_transition.blockchain_length block , External_transition.global_slot block ) type valid_payload = { state_hash : State_hash.t + ; parent : State_hash.t ; height : Unsigned.uint32 ; slot : Unsigned.uint32 } -let valid_payload_to_yojson { state_hash; height; slot } : Yojson.Safe.t = +let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t + = `Assoc [ ("state_hash", State_hash.to_yojson state_hash) + ; ("parent", State_hash.to_yojson parent) ; ("height", `Int (Unsigned.UInt32.to_int height)) ; ("slot", `Int (Unsigned.UInt32.to_int slot)) ] @@ -133,14 +139,22 @@ let command = flag "--block-dir" ~aliases:[ "-block-dir" ] ~doc:"the path to the directory containing blocks for the submission" (required Filename.arg_type) - and inputs = anon (sequence ("filename" %: Filename.arg_type)) in + and inputs = anon (sequence ("filename" %: Filename.arg_type)) + and no_checks = + flag "--no-checks" ~aliases:[ "-no-checks" ] + ~doc: + "disable all the checks, just extract the info from the submissions" + no_arg + in fun () -> let open Deferred.Let_syntax in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> - match%bind validate_submission ~block_dir ~metadata_path with - | Ok (state_hash, height, slot) -> - display { state_hash; height; slot } ; + match%bind + validate_submission ~block_dir ~metadata_path ~no_checks + with + | Ok (state_hash, parent, height, slot) -> + display { state_hash; parent; height; slot } ; Deferred.unit | Error `Path_is_invalid -> display_error "path for metadata is invalid" ; From f77c1848190ebab4507e85c7531ff0c4bc511f72 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 26 Jan 2022 22:37:03 +0800 Subject: [PATCH 049/111] fix typo --- src/app/delegation_verify/delegation_verify.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 4e2c8d78afae..795af92059a7 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -104,7 +104,7 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = verify_snark_work ~proof ~message in ( External_transition.state_hash block - , External_transition.parent block + , External_transition.parent_hash block , External_transition.blockchain_length block , External_transition.global_slot block ) From 43d7b7e8ae01f22bfc2c79308946b726d84fcab3 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Wed, 26 Jan 2022 23:39:08 +0800 Subject: [PATCH 050/111] delay the creation of the verifier --- .../delegation_verify/delegation_verify.ml | 10 ++++---- src/app/delegation_verify/verifier.ml | 24 +++++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 795af92059a7..15fe9cb9128a 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -57,11 +57,10 @@ let decode_block str = let verify_block ~block = let open External_transition in + let verify_blockchain_snarks = force Verifier.verify_blockchain_snarks in let%map result = - Verifier.verify_blockchain_snarks - [ Blockchain_snark.Blockchain.create ~state:(protocol_state block) - ~proof:(protocol_state_proof block) - ] + verify_blockchain_snarks + [ (protocol_state block, protocol_state_proof block) ] in if result then Ok () else Error `Invalid_proof @@ -74,7 +73,8 @@ let decode_snark_work str = Error `Fail_to_decode_snark_work let verify_snark_work ~proof ~message = - let%map result = Verifier.verify_transaction_snarks [ (proof, message) ] in + let verify_transaction_snarks = force Verifier.verify_transaction_snarks in + let%map result = verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work let validate_submission ~block_dir ~metadata_path ~no_checks = diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index aa95cebf85f0..7762e481aa44 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -14,10 +14,30 @@ module B = Blockchain_snark.Blockchain_snark_state.Make (struct let proof_level = Genesis_constants.Proof_level.compiled end) -let verify_blockchain_snarks (chains : Blockchain_snark.Blockchain.t list) = +let verify_blockchain_snarks_ (chains : Blockchain_snark.Blockchain.t list) = B.Proof.verify @@ List.map chains ~f:(fun snark -> ( Blockchain_snark.Blockchain.state snark , Blockchain_snark.Blockchain.proof snark )) -let verify_transaction_snarks ts = T.verify ts +let verify_blockchain_snarks = + lazy + (let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = + Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled + end) in + B.Proof.verify) + +let verify_transaction_snarks = + lazy + (let module T = Transaction_snark.Make (struct + let constraint_constants = + Genesis_constants.Constraint_constants.compiled + + let proof_level = Genesis_constants.Proof_level.compiled + end) in + T.verify) From 4858ef87e8386c0c8dfacc9eb3416b67cea826b7 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 27 Jan 2022 00:04:23 +0800 Subject: [PATCH 051/111] wrap verification in a pair of functions --- .../delegation_verify/delegation_verify.ml | 4 +- src/app/delegation_verify/verifier.ml | 43 ++++--------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 15fe9cb9128a..8518997fd61d 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -57,7 +57,7 @@ let decode_block str = let verify_block ~block = let open External_transition in - let verify_blockchain_snarks = force Verifier.verify_blockchain_snarks in + let verify_blockchain_snarks, _ = force Verifier.verify_functions in let%map result = verify_blockchain_snarks [ (protocol_state block, protocol_state_proof block) ] @@ -73,7 +73,7 @@ let decode_snark_work str = Error `Fail_to_decode_snark_work let verify_snark_work ~proof ~message = - let verify_transaction_snarks = force Verifier.verify_transaction_snarks in + let _, verify_transaction_snarks = force Verifier.verify_functions in let%map result = verify_transaction_snarks [ (proof, message) ] in if result then Ok () else Error `Invalid_snark_work diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index 7762e481aa44..852ec044d983 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -1,43 +1,16 @@ -open Core - -module T = Transaction_snark.Make (struct - let constraint_constants = Genesis_constants.Constraint_constants.compiled - - let proof_level = Genesis_constants.Proof_level.compiled -end) - -module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - - let constraint_constants = Genesis_constants.Constraint_constants.compiled - - let proof_level = Genesis_constants.Proof_level.compiled -end) - -let verify_blockchain_snarks_ (chains : Blockchain_snark.Blockchain.t list) = - B.Proof.verify - @@ List.map chains ~f:(fun snark -> - ( Blockchain_snark.Blockchain.state snark - , Blockchain_snark.Blockchain.proof snark )) - -let verify_blockchain_snarks = +let verify_functions = lazy - (let module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - + (let module T = Transaction_snark.Make (struct let constraint_constants = Genesis_constants.Constraint_constants.compiled let proof_level = Genesis_constants.Proof_level.compiled end) in - B.Proof.verify) + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag -let verify_transaction_snarks = - lazy - (let module T = Transaction_snark.Make (struct - let constraint_constants = - Genesis_constants.Constraint_constants.compiled + let constraint_constants = Genesis_constants.Constraint_constants.compiled - let proof_level = Genesis_constants.Proof_level.compiled - end) in - T.verify) + let proof_level = Genesis_constants.Proof_level.compiled + end) in + (B.Proof.verify, T.verify)) From 3d2c34d2d67953c2075649de4954ee458f4a7c85 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 27 Jan 2022 00:29:58 +0800 Subject: [PATCH 052/111] remove rpc_parallel stuff --- src/app/delegation_verify/delegation_verify.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 8518997fd61d..bd80db06c55e 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -179,4 +179,4 @@ let command = display_error "fail to verify the snark work" ; Deferred.unit)) -let () = Rpc_parallel.start_app command +let () = Async.Command.run command From eb71b7a846e31f5715f4a156f2797c751e310e94 Mon Sep 17 00:00:00 2001 From: georgeee Date: Fri, 8 Sep 2023 20:39:41 +0200 Subject: [PATCH 053/111] Fix after rebase onto latest compatible --- .../Dockerfile-delegation-stateless-verifier | 2 +- .../delegation_verify/delegation_verify.ml | 67 ++++++++++--------- src/app/delegation_verify/dune | 41 +++++++++++- src/app/delegation_verify/verifier.ml | 2 +- 4 files changed, 75 insertions(+), 37 deletions(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 8a6adc76f6e5..e607dc6b0efe 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -3,7 +3,7 @@ # - builds stateless verification tool # - should not include any data related to joining a specific network, only the node software itself ################################################################################################# -FROM gcr.io/o1labs-192920/mina-toolchain@sha256:7ef5827fa0854a0bcfdee69dcc0c2c7aef86e1662c630e71f07c1f9162e757fa AS builder +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:73562fcc35dcabd342f66f1d69ae12704e92d69edc0b37e7c88b4d11bc623f23 AS builder # Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI ARG DUNE_PROFILE=devnet diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index bd80db06c55e..6ca67887ae47 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -1,5 +1,4 @@ open Mina_base -open Mina_transition open Core open Async open Signature_lib @@ -30,39 +29,35 @@ let load_metadata str = (* This decoding is also unnecessarily complicated given that we are the creator of those data *) let decode_metadata str = - match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ok - { created_at = _ - ; peer_id = _ - ; snark_work - ; remote_addr = _ - ; submitter - ; block_hash - } -> ( - match submitter |> Public_key.Compressed.of_base58_check with - | Ok submitter -> - Ok (snark_work, submitter, block_hash) - | Error _ -> - Error `Fail_to_decode_metadata ) - | Error _ -> - Error `Fail_to_decode_metadata + let parsed_meta = + match Yojson.Safe.from_string str |> metadata_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error _ -> + Error `Fail_to_decode_metadata + in + let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in + let%bind.Result submitter = + Result.map_error ~f:(const `Fail_to_decode_metadata) + @@ Public_key.Compressed.of_base58_check submitter + in + Ok (snark_work, submitter, block_hash) let load_block ~block_dir ~block_hash = let block_path = Filename.concat block_dir (block_hash ^ ".dat") in try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block let decode_block str = - try Ok (Binable.of_string (module External_transition.Stable.Latest) str) + try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) with _ -> Error `Fail_to_decode_block let verify_block ~block = - let open External_transition in + let header = Mina_block.header block in + let open Mina_block.Header in let verify_blockchain_snarks, _ = force Verifier.verify_functions in - let%map result = - verify_blockchain_snarks - [ (protocol_state block, protocol_state_proof block) ] - in - if result then Ok () else Error `Invalid_proof + verify_blockchain_snarks + [ (protocol_state header, protocol_state_proof header) ] + |> Deferred.Result.map_error ~f:(const `Invalid_proof) let decode_snark_work str = match Base64.decode str with @@ -74,8 +69,8 @@ let decode_snark_work str = let verify_snark_work ~proof ~message = let _, verify_transaction_snarks = force Verifier.verify_functions in - let%map result = verify_transaction_snarks [ (proof, message) ] in - if result then Ok () else Error `Invalid_snark_work + verify_transaction_snarks [ (proof, message) ] + |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) let validate_submission ~block_dir ~metadata_path ~no_checks = let open Deferred.Result.Let_syntax in @@ -103,16 +98,22 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = in verify_snark_work ~proof ~message in - ( External_transition.state_hash block - , External_transition.parent_hash block - , External_transition.blockchain_length block - , External_transition.global_slot block ) + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) type valid_payload = { state_hash : State_hash.t ; parent : State_hash.t ; height : Unsigned.uint32 - ; slot : Unsigned.uint32 + ; slot : Mina_numbers.Global_slot_since_genesis.t } let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t @@ -121,7 +122,7 @@ let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t [ ("state_hash", State_hash.to_yojson state_hash) ; ("parent", State_hash.to_yojson parent) ; ("height", `Int (Unsigned.UInt32.to_int height)) - ; ("slot", `Int (Unsigned.UInt32.to_int slot)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int slot)) ] let display valid_payload = @@ -177,6 +178,6 @@ let command = Deferred.unit | Error `Invalid_snark_work -> display_error "fail to verify the snark work" ; - Deferred.unit)) + Deferred.unit )) let () = Async.Command.run command diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 7df3a9b2c5dd..1e7bc90927bc 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,5 +1,42 @@ (executable (name delegation_verify) - (libraries uptime_service core_kernel async mina_transition parallel) + (libraries + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base_import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_backend.pasta + kimchi_backend.pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + ) (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_coda ppx_version ppx_jane ppx_custom_printf h_list.ppx))) + (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index 852ec044d983..fd8c5d66919f 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -13,4 +13,4 @@ let verify_functions = let proof_level = Genesis_constants.Proof_level.compiled end) in - (B.Proof.verify, T.verify)) + (B.Proof.verify, T.verify) ) From 05c274307a5d8c51377a975e79f4198dc854c7e5 Mon Sep 17 00:00:00 2001 From: georgeee Date: Fri, 8 Sep 2023 23:10:38 +0200 Subject: [PATCH 054/111] Support graphql_control_port --- src/app/delegation_verify/delegation_verify.ml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6ca67887ae47..76dc7a61ab14 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,6 +10,7 @@ type metadata = ; remote_addr : string ; submitter : string ; block_hash : string + ; graphql_control_port: int option [@default None] } [@@deriving yojson] @@ -33,12 +34,12 @@ let decode_metadata str = match Yojson.Safe.from_string str |> metadata_of_yojson with | Ppx_deriving_yojson_runtime.Result.Ok a -> Ok a - | Ppx_deriving_yojson_runtime.Result.Error _ -> - Error `Fail_to_decode_metadata + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (`Fail_to_decode_metadata e) in let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in let%bind.Result submitter = - Result.map_error ~f:(const `Fail_to_decode_metadata) + Result.map_error ~f:(fun e -> `Fail_to_decode_metadata (Error.to_string_hum e)) @@ Public_key.Compressed.of_base58_check submitter in Ok (snark_work, submitter, block_hash) @@ -160,9 +161,12 @@ let command = | Error `Path_is_invalid -> display_error "path for metadata is invalid" ; exit 1 - | Error `Fail_to_load_metadata | Error `Fail_to_decode_metadata -> + | Error `Fail_to_load_metadata -> display_error "fail to load metadata" ; exit 2 + | Error (`Fail_to_decode_metadata e) -> + display_error ("fail to decode metadata: " ^ e); + exit 2 | Error `Fail_to_load_block -> display_error "fail to load block" ; exit 3 From a39e7e7f1471c6f91b50114eac7ab0451a696735 Mon Sep 17 00:00:00 2001 From: georgeee Date: Sun, 10 Sep 2023 15:37:30 +0200 Subject: [PATCH 055/111] Add --config-file optional argument Problem: delegation stateless verification tool is not usable for submissions on cluster with custum constraints constants. Solution: provide an option to load config file with custom constraints constants. Bonus: code prettified and reformated. --- .../delegation_verify/delegation_verify.ml | 94 ++++++++++++++----- src/app/delegation_verify/dune | 4 + src/app/delegation_verify/verifier.ml | 24 +++-- 3 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 76dc7a61ab14..b7c6650f1202 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,7 +10,7 @@ type metadata = ; remote_addr : string ; submitter : string ; block_hash : string - ; graphql_control_port: int option [@default None] + ; graphql_control_port : int option [@default None] } [@@deriving yojson] @@ -22,9 +22,6 @@ let get_filenames = | filenames -> filenames -(* This check seems unnecessary if the submission data is published by ourselfes *) -let check_path _ = Ok () - let load_metadata str = try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata @@ -39,7 +36,8 @@ let decode_metadata str = in let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in let%bind.Result submitter = - Result.map_error ~f:(fun e -> `Fail_to_decode_metadata (Error.to_string_hum e)) + Result.map_error ~f:(fun e -> + `Fail_to_decode_metadata (Error.to_string_hum e) ) @@ Public_key.Compressed.of_base58_check submitter in Ok (snark_work, submitter, block_hash) @@ -52,10 +50,9 @@ let decode_block str = try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) with _ -> Error `Fail_to_decode_block -let verify_block ~block = +let verify_block ~verify_blockchain_snarks ~block = let header = Mina_block.header block in let open Mina_block.Header in - let verify_blockchain_snarks, _ = force Verifier.verify_functions in verify_blockchain_snarks [ (protocol_state header, protocol_state_proof header) ] |> Deferred.Result.map_error ~f:(const `Invalid_proof) @@ -68,21 +65,23 @@ let decode_snark_work str = | Error _ -> Error `Fail_to_decode_snark_work -let verify_snark_work ~proof ~message = - let _, verify_transaction_snarks = force Verifier.verify_functions in +let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -let validate_submission ~block_dir ~metadata_path ~no_checks = +let validate_submission ~block_dir ~metadata_path ~no_checks + (verify_blockchain_snarks, verify_transaction_snarks) = let open Deferred.Result.Let_syntax in - let%bind () = Deferred.return @@ check_path metadata_path in let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in let%bind snark_work_opt, submitter, block_hash = Deferred.return @@ decode_metadata metadata_str in let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = if no_checks then return () else verify_block ~block in + let%bind () = + if no_checks then return () + else verify_block ~verify_blockchain_snarks ~block + in let%map () = if no_checks then return () else @@ -97,7 +96,7 @@ let validate_submission ~block_dir ~metadata_path ~no_checks = let message = Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter in - verify_snark_work ~proof ~message + verify_snark_work ~verify_transaction_snarks ~proof ~message in let header = Mina_block.header block in let protocol_state = Mina_block.Header.protocol_state header in @@ -133,27 +132,72 @@ let display valid_payload = let display_error e = eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] +let config_flag = + let open Command.Param in + flag "--config-file" ~doc:"FILE config file" (optional string) + +let no_checks_flag = + let open Command.Param in + flag "--no-checks" ~aliases:[ "-no-checks" ] + ~doc:"disable all the checks, just extract the info from the submissions" + no_arg + +let block_dir_flag = + let open Command.Param in + flag "--block-dir" ~aliases:[ "-block-dir" ] + ~doc:"the path to the directory containing blocks for the submission" + (required Filename.arg_type) + +let instantiate_verify_functions ~logger = function + | None -> + Deferred.return + (Verifier.verify_functions + ~constraint_constants:Genesis_constants.Constraint_constants.compiled + ~proof_level:Genesis_constants.Proof_level.compiled () ) + | Some config_file -> + let%bind.Deferred precomputed_values = + let%bind.Deferred.Or_error config_json = + Genesis_ledger_helper.load_config_json config_file + in + let%bind.Deferred.Or_error config = + Deferred.return + @@ Result.map_error ~f:Error.of_string + @@ Runtime_config.of_yojson config_json + in + Genesis_ledger_helper.init_from_config_file ~logger ~proof_level:None + config + in + let%map.Deferred precomputed_values = + match precomputed_values with + | Ok (precomputed_values, _) -> + Deferred.return precomputed_values + | Error _ -> + display_error "fail to read config file" ; + exit 4 + in + let constraint_constants = + Precomputed_values.constraint_constants precomputed_values + in + Verifier.verify_functions ~constraint_constants ~proof_level:Full () + let command = Command.async ~summary:"A tool for verifying JSON payload submitted by the uptime service" Command.Let_syntax.( - let%map_open block_dir = - flag "--block-dir" ~aliases:[ "-block-dir" ] - ~doc:"the path to the directory containing blocks for the submission" - (required Filename.arg_type) + let%map_open block_dir = block_dir_flag and inputs = anon (sequence ("filename" %: Filename.arg_type)) - and no_checks = - flag "--no-checks" ~aliases:[ "-no-checks" ] - ~doc: - "disable all the checks, just extract the info from the submissions" - no_arg - in + and no_checks = no_checks_flag + and config_file = config_flag in fun () -> + let logger = Logger.create () in + let%bind.Deferred ver_funs = + instantiate_verify_functions ~logger config_file + in let open Deferred.Let_syntax in let metadata_pathes = get_filenames inputs in Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> match%bind - validate_submission ~block_dir ~metadata_path ~no_checks + validate_submission ~block_dir ~metadata_path ~no_checks ver_funs with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; @@ -165,7 +209,7 @@ let command = display_error "fail to load metadata" ; exit 2 | Error (`Fail_to_decode_metadata e) -> - display_error ("fail to decode metadata: " ^ e); + display_error ("fail to decode metadata: " ^ e) ; exit 2 | Error `Fail_to_load_block -> display_error "fail to load block" ; diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 1e7bc90927bc..bf8815b7b31c 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -37,6 +37,10 @@ pickles pickles_types pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger ) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml index fd8c5d66919f..43677f5d6405 100644 --- a/src/app/delegation_verify/verifier.ml +++ b/src/app/delegation_verify/verifier.ml @@ -1,16 +1,14 @@ -let verify_functions = - lazy - (let module T = Transaction_snark.Make (struct - let constraint_constants = - Genesis_constants.Constraint_constants.compiled +let verify_functions ~constraint_constants ~proof_level () = + let module T = Transaction_snark.Make (struct + let constraint_constants = constraint_constants - let proof_level = Genesis_constants.Proof_level.compiled - end) in - let module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag + let proof_level = proof_level + end) in + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag - let constraint_constants = Genesis_constants.Constraint_constants.compiled + let constraint_constants = constraint_constants - let proof_level = Genesis_constants.Proof_level.compiled - end) in - (B.Proof.verify, T.verify) ) + let proof_level = proof_level + end) in + (B.Proof.verify, T.verify) From d4a72bcdfe62ac406caee95d5a55f7f80baa588f Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 9 Nov 2023 10:59:51 +0100 Subject: [PATCH 056/111] [delegation_verify] Refactor in order to abstract over input loading. --- .../delegation_verify/delegation_verify.ml | 207 ++++++++---------- src/app/delegation_verify/submission.ml | 89 ++++++++ 2 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 src/app/delegation_verify/submission.ml diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index b7c6650f1202..44121ca45f62 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -1,18 +1,19 @@ open Mina_base open Core open Async -open Signature_lib - -type metadata = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } -[@@deriving yojson] + +type error = + [ `Path_is_invalid + | `Fail_to_load_metadata + | `Fail_to_decode_metadata of string + | `Fail_to_load_block + | `Fail_to_decode_block + | `Invalid_proof + | `Fail_to_decode_snark_work + | `Invalid_snark_work + ] + +let unify_error e = (e :> error) let get_filenames = let open In_channel in @@ -22,34 +23,6 @@ let get_filenames = | filenames -> filenames -let load_metadata str = - try Ok (In_channel.read_all str) with _ -> Error `Fail_to_load_metadata - -(* This decoding is also unnecessarily complicated given that we are the creator of those data *) -let decode_metadata str = - let parsed_meta = - match Yojson.Safe.from_string str |> metadata_of_yojson with - | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a - | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (`Fail_to_decode_metadata e) - in - let%bind.Result { snark_work; submitter; block_hash; _ } = parsed_meta in - let%bind.Result submitter = - Result.map_error ~f:(fun e -> - `Fail_to_decode_metadata (Error.to_string_hum e) ) - @@ Public_key.Compressed.of_base58_check submitter - in - Ok (snark_work, submitter, block_hash) - -let load_block ~block_dir ~block_hash = - let block_path = Filename.concat block_dir (block_hash ^ ".dat") in - try Ok (In_channel.read_all block_path) with _ -> Error `Fail_to_load_block - -let decode_block str = - try Ok (Binable.of_string (module Mina_block.Stable.Latest) str) - with _ -> Error `Fail_to_decode_block - let verify_block ~verify_blockchain_snarks ~block = let header = Mina_block.header block in let open Mina_block.Header in @@ -57,57 +30,48 @@ let verify_block ~verify_blockchain_snarks ~block = [ (protocol_state header, protocol_state_proof header) ] |> Deferred.Result.map_error ~f:(const `Invalid_proof) -let decode_snark_work str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error `Fail_to_decode_snark_work ) - | Error _ -> - Error `Fail_to_decode_snark_work - let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -let validate_submission ~block_dir ~metadata_path ~no_checks - (verify_blockchain_snarks, verify_transaction_snarks) = - let open Deferred.Result.Let_syntax in - let%bind metadata_str = Deferred.return @@ load_metadata metadata_path in - let%bind snark_work_opt, submitter, block_hash = - Deferred.return @@ decode_metadata metadata_str - in - let%bind block_str = Deferred.return @@ load_block ~block_dir ~block_hash in - let%bind block = Deferred.return @@ decode_block block_str in - let%bind () = - if no_checks then return () - else verify_block ~verify_blockchain_snarks ~block - in - let%map () = - if no_checks then return () - else - match snark_work_opt with - | None -> - Deferred.Result.return () - | Some snark_work_str -> - let%bind Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } = - Deferred.return @@ decode_snark_work snark_work_str - in - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:submitter - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - in - let header = Mina_block.header block in - let protocol_state = Mina_block.Header.protocol_state header in - let consensus_state = - Mina_state.Protocol_state.consensus_state protocol_state - in - ( Mina_state.Protocol_state.hashes protocol_state - |> State_hash.State_hashes.state_hash - , Mina_state.Protocol_state.previous_state_hash protocol_state - , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) +module Validator(Sub : Submission.S) = struct + let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission = + let open Deferred.Result.Let_syntax in + let%bind block = + Sub.block submission + |> Result.map_error ~f:unify_error + |> Deferred.return + in + let%bind () = + if no_checks then return () + else verify_block ~verify_blockchain_snarks ~block + |> Deferred.Result.map_error ~f:unify_error + in + let%map () = + if no_checks then return () + else + match Sub.snark_work submission with + | None -> + Deferred.Result.return () + | Some Uptime_service.Proof_data. + { proof; proof_time = _; snark_work_fee } -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:(Sub.submitter submission) + in + verify_snark_work ~verify_transaction_snarks ~proof ~message + |> Deferred.Result.map_error ~f:unify_error + in + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) +end type valid_payload = { state_hash : State_hash.t @@ -180,6 +144,33 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () +let handle_error = function + | `Path_is_invalid -> + display_error "path for metadata is invalid" ; + exit 1 + | `Fail_to_load_metadata -> + display_error "fail to load metadata" ; + exit 2 + | `Fail_to_decode_metadata e -> + display_error ("fail to decode metadata: " ^ e) ; + exit 2 + | `Fail_to_load_block -> + display_error "fail to load block" ; + exit 3 + | `Fail_to_decode_block -> + display_error "fail to decode block" ; + Deferred.unit + | `Invalid_proof -> + display_error + "fail to verify the protocol state proof inside the block" ; + Deferred.unit + | `Fail_to_decode_snark_work -> + display_error "fail to decode snark work" ; + Deferred.unit + | `Invalid_snark_work -> + display_error "fail to verify the snark work" ; + Deferred.unit + let command = Command.async ~summary:"A tool for verifying JSON payload submitted by the uptime service" @@ -190,42 +181,26 @@ let command = and config_file = config_flag in fun () -> let logger = Logger.create () in - let%bind.Deferred ver_funs = + let%bind.Deferred (verify_blockchain_snarks, verify_transaction_snarks) = instantiate_verify_functions ~logger config_file in let open Deferred.Let_syntax in - let metadata_pathes = get_filenames inputs in - Deferred.List.iter metadata_pathes ~f:(fun metadata_path -> + let metadata_paths = get_filenames inputs in + let module V = Validator(Submission.JSON) in + Deferred.List.iter metadata_paths ~f:(fun path -> match%bind - validate_submission ~block_dir ~metadata_path ~no_checks ver_funs + ( let open Deferred.Result.Let_syntax in + let%bind submission = + Submission.JSON.load ~block_dir path + |> Result.map_error ~f:unify_error + |> Deferred.return + in + V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; Deferred.unit - | Error `Path_is_invalid -> - display_error "path for metadata is invalid" ; - exit 1 - | Error `Fail_to_load_metadata -> - display_error "fail to load metadata" ; - exit 2 - | Error (`Fail_to_decode_metadata e) -> - display_error ("fail to decode metadata: " ^ e) ; - exit 2 - | Error `Fail_to_load_block -> - display_error "fail to load block" ; - exit 3 - | Error `Fail_to_decode_block -> - display_error "fail to decode block" ; - Deferred.unit - | Error `Invalid_proof -> - display_error - "fail to verify the protocol state proof inside the block" ; - Deferred.unit - | Error `Fail_to_decode_snark_work -> - display_error "fail to decode snark work" ; - Deferred.unit - | Error `Invalid_snark_work -> - display_error "fail to verify the snark work" ; - Deferred.unit )) + | Error e -> + handle_error e)) let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml new file mode 100644 index 000000000000..5df847e4b0c3 --- /dev/null +++ b/src/app/delegation_verify/submission.ml @@ -0,0 +1,89 @@ +open Core +open Signature_lib + + type block_error = + [ `Fail_to_load_block + | `Fail_to_decode_block + ] + +module type S = sig + type t + + val snark_work : t -> Uptime_service.Proof_data.t option + + val submitter : t -> Public_key.Compressed.t + + val block_hash : t -> string + + val block : t -> (Mina_block.t, block_error) Result.t +end + + +module JSON = struct + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } + [@@deriving yojson] + + type t = + { snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + ; block_dir : string + } + + let snark_work { snark_work; _ } = snark_work + let submitter { submitter; _ } = submitter + let block_hash { block_hash; _ } = block_hash + + let block { block_dir; block_hash; _ } = + let open Result.Let_syntax in + let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in + let%bind contents = + try Ok (In_channel.read_all block_path) with + _ -> Error `Fail_to_load_block + in + try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) + with _ -> Error `Fail_to_decode_block + + let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error `Fail_to_decode_snark_work ) + | Error _ -> + Error `Fail_to_decode_snark_work + + let load ~block_dir filename = + let open Result.Let_syntax in + let%bind contents = + try Ok (In_channel.read_all filename) with _ -> Error `Fail_to_load_metadata + in + let%bind meta = + match Yojson.Safe.from_string contents |> raw_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (`Fail_to_decode_metadata e) + in + let%bind.Result submitter = + Result.map_error ~f:(fun e -> + `Fail_to_decode_metadata (Error.to_string_hum e) ) + @@ Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash; block_dir } +end From d786cc4f1e4434f6e5bbe71dedb4f6f3e4cd4f82 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 10 Nov 2023 16:38:50 +0100 Subject: [PATCH 057/111] [delegation_verify] Refactor error handling. --- .../delegation_verify/delegation_verify.ml | 83 +++++-------------- src/app/delegation_verify/submission.ml | 48 +++++------ 2 files changed, 41 insertions(+), 90 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 44121ca45f62..af505f286fa3 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -2,19 +2,6 @@ open Mina_base open Core open Async -type error = - [ `Path_is_invalid - | `Fail_to_load_metadata - | `Fail_to_decode_metadata of string - | `Fail_to_load_block - | `Fail_to_decode_block - | `Invalid_proof - | `Fail_to_decode_snark_work - | `Invalid_snark_work - ] - -let unify_error e = (e :> error) - let get_filenames = let open In_channel in function @@ -28,38 +15,33 @@ let verify_block ~verify_blockchain_snarks ~block = let open Mina_block.Header in verify_blockchain_snarks [ (protocol_state header, protocol_state_proof header) ] - |> Deferred.Result.map_error ~f:(const `Invalid_proof) let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] - |> Deferred.Result.map_error ~f:(const `Invalid_snark_work) -module Validator(Sub : Submission.S) = struct - let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission = +module Validator (Sub : Submission.S) = struct + let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks + submission = let open Deferred.Result.Let_syntax in - let%bind block = - Sub.block submission - |> Result.map_error ~f:unify_error - |> Deferred.return - in + let%bind block = Sub.block submission |> Deferred.return in let%bind () = if no_checks then return () else verify_block ~verify_blockchain_snarks ~block - |> Deferred.Result.map_error ~f:unify_error in let%map () = if no_checks then return () else match Sub.snark_work submission with | None -> - Deferred.Result.return () - | Some Uptime_service.Proof_data. - { proof; proof_time = _; snark_work_fee } -> - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee ~prover:(Sub.submitter submission) - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - |> Deferred.Result.map_error ~f:unify_error + Deferred.Result.return () + | Some + Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } + -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:(Sub.submitter submission) + in + verify_snark_work ~verify_transaction_snarks ~proof ~message in let header = Mina_block.header block in let protocol_state = Mina_block.Header.protocol_state header in @@ -70,7 +52,8 @@ module Validator(Sub : Submission.S) = struct |> State_hash.State_hashes.state_hash , Mina_state.Protocol_state.previous_state_hash protocol_state , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state + ) end type valid_payload = @@ -144,36 +127,8 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () -let handle_error = function - | `Path_is_invalid -> - display_error "path for metadata is invalid" ; - exit 1 - | `Fail_to_load_metadata -> - display_error "fail to load metadata" ; - exit 2 - | `Fail_to_decode_metadata e -> - display_error ("fail to decode metadata: " ^ e) ; - exit 2 - | `Fail_to_load_block -> - display_error "fail to load block" ; - exit 3 - | `Fail_to_decode_block -> - display_error "fail to decode block" ; - Deferred.unit - | `Invalid_proof -> - display_error - "fail to verify the protocol state proof inside the block" ; - Deferred.unit - | `Fail_to_decode_snark_work -> - display_error "fail to decode snark work" ; - Deferred.unit - | `Invalid_snark_work -> - display_error "fail to verify the snark work" ; - Deferred.unit - -let command = - Command.async - ~summary:"A tool for verifying JSON payload submitted by the uptime service" +let filesystem_command = + Command.async ~summary:"Verify submissions and block read from the filesystem" Command.Let_syntax.( let%map_open block_dir = block_dir_flag and inputs = anon (sequence ("filename" %: Filename.arg_type)) @@ -192,7 +147,6 @@ let command = ( let open Deferred.Result.Let_syntax in let%bind submission = Submission.JSON.load ~block_dir path - |> Result.map_error ~f:unify_error |> Deferred.return in V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) @@ -201,6 +155,7 @@ let command = display { state_hash; parent; height; slot } ; Deferred.unit | Error e -> - handle_error e)) + display_error @@ Error.to_string_hum e ; + Deferred.unit)) let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5df847e4b0c3..09247d47f0e2 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -1,11 +1,6 @@ open Core open Signature_lib - type block_error = - [ `Fail_to_load_block - | `Fail_to_decode_block - ] - module type S = sig type t @@ -15,12 +10,10 @@ module type S = sig val block_hash : t -> string - val block : t -> (Mina_block.t, block_error) Result.t + val block : t -> Mina_block.t Or_error.t end - module JSON = struct - type raw = { created_at : string ; peer_id : string @@ -30,7 +23,7 @@ module JSON = struct ; block_hash : string ; graphql_control_port : int option [@default None] } - [@@deriving yojson] + [@@deriving yojson] type t = { snark_work : Uptime_service.Proof_data.t option @@ -40,50 +33,53 @@ module JSON = struct } let snark_work { snark_work; _ } = snark_work + let submitter { submitter; _ } = submitter + let block_hash { block_hash; _ } = block_hash let block { block_dir; block_hash; _ } = let open Result.Let_syntax in let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in let%bind contents = - try Ok (In_channel.read_all block_path) with - _ -> Error `Fail_to_load_block + try Ok (In_channel.read_all block_path) + with _ -> Error (Error.of_string "Fail to load block") in try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) - with _ -> Error `Fail_to_decode_block + with _ -> Error (Error.of_string "Fail to decode block") let decode_snark_work str = match Base64.decode str with | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error `Fail_to_decode_snark_work ) + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error (Error.of_string "Fail to decode snark work") ) | Error _ -> - Error `Fail_to_decode_snark_work + Error (Error.of_string "Fail to decode snark work") let load ~block_dir filename = let open Result.Let_syntax in let%bind contents = - try Ok (In_channel.read_all filename) with _ -> Error `Fail_to_load_metadata + try Ok (In_channel.read_all filename) + with _ -> Error (Error.of_string "Fail to load metadata") in let%bind meta = match Yojson.Safe.from_string contents |> raw_of_yojson with | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a + Ok a | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (`Fail_to_decode_metadata e) + Error (Error.of_string e) in let%bind.Result submitter = - Result.map_error ~f:(fun e -> - `Fail_to_decode_metadata (Error.to_string_hum e) ) - @@ Public_key.Compressed.of_base58_check meta.submitter + Public_key.Compressed.of_base58_check meta.submitter in let%map snark_work = - match meta.snark_work with - | None -> Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work in { submitter; snark_work; block_hash = meta.block_hash; block_dir } end + From 167c5a8234c682097cfa08ae71eb221dce6ee31e Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 13 Nov 2023 14:42:49 +0100 Subject: [PATCH 058/111] Add an option to read data from Cassandra (unfinished). The setup depends on ~/.cassandra/cqlshrc configuration to work. --- src/app/delegation_verify/cassandra.ml | 81 +++++++++++++++++++ src/app/delegation_verify/cassandra.mli | 10 +++ .../delegation_verify/delegation_verify.ml | 61 +++++++++++--- src/app/delegation_verify/submission.ml | 1 - 4 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/app/delegation_verify/cassandra.ml create mode 100644 src/app/delegation_verify/cassandra.mli diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml new file mode 100644 index 000000000000..4acacae6a735 --- /dev/null +++ b/src/app/delegation_verify/cassandra.ml @@ -0,0 +1,81 @@ +open Async +open Core + +type connection = Process.t + +type 'a t = connection -> 'a Or_error.t Deferred.t + +module M = struct + type nonrec 'a t = 'a t + + let return x _ = return (Ok x) + + let map = `Custom (fun m ~f c -> m c >>| Or_error.map ~f) + + let bind m ~f c = + m c >>= function Error _ as e -> Deferred.return e | Ok x -> f x c +end + +include Monad.Make (M) + +let lift m _ = m + +(* Open connection to a Cassandra database using cqlsh executable + provided. The connection should be configured via ~/.cassandra/cqlshrc + configuration file. Return the process handle which can the be used to + send queries to the database. *) +let connect ?executable keyspace = + let open Deferred.Or_error.Let_syntax in + let prog = + Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const + |> Option.value ~default:"cqlsh" + in + let%map proc = Process.create ~prog ~args:[ "-k"; keyspace ] () in + Async.printf "Connected to the Cassandra database\n" ; + proc + +let exec ?cqlsh ~keyspace m = + let open Deferred.Or_error.Monad_infix in + connect ?executable:cqlsh keyspace >>= m + +let read_json_line chan = + let open Deferred.Let_syntax in + match%map Reader.read_line chan with + | `Eof -> + Or_error.error_string "Cassandra client: broken pipe" + | `Ok line -> ( + try + let j = Yojson.Safe.from_string line in + match Submission.JSON.raw_of_yojson j with + | Ppx_deriving_yojson_runtime.Result.Ok s -> + Ok s + | Ppx_deriving_yojson_runtime.Result.Error e -> + Or_error.error_string e + with Yojson.Json_error e -> Or_error.error_string e ) + +let rec read_json_output acc chan = + let open Deferred.Or_error.Let_syntax in + (* skip empty line *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + (* [json] header *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + (* [json] header *) + let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with + | Ok s -> + read_json_output (s :: acc) chan + | Error _ -> + return (List.rev acc) + +let query q conn = + let open Deferred.Let_syntax in + let proc_stdin = Process.stdin conn in + Writer.write_line proc_stdin q ; + Writer.(write_line @@ Lazy.force stdout) q ; + let%bind () = Writer.close proc_stdin in + let%bind () = Writer.flushed proc_stdin in + match%bind read_json_output [] (Process.stdout conn) with + | Ok output -> + return (Ok output) + | Error e -> + return (Error e) diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli new file mode 100644 index 000000000000..c1e30f7e977a --- /dev/null +++ b/src/app/delegation_verify/cassandra.mli @@ -0,0 +1,10 @@ +open Async +open Core + +include Monad.S + +val lift : 'a Deferred.Or_error.t -> 'a t + +val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t + +val query : string -> Submission.JSON.raw list t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index af505f286fa3..8e41eab5c4aa 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -95,6 +95,17 @@ let block_dir_flag = ~doc:"the path to the directory containing blocks for the submission" (required Filename.arg_type) +let cassandra_executable_flag = + let open Command.Param in + flag "--executable" + ~aliases:[ "-executable"; "--cqlsh"; "-cqlsh" ] + ~doc:"the path to the cqlsh executable" + (optional Filename.arg_type) + +let timestamp = + let open Command.Param in + anon ("timestamp" %: string) + let instantiate_verify_functions ~logger = function | None -> Deferred.return @@ -136,26 +147,56 @@ let filesystem_command = and config_file = config_flag in fun () -> let logger = Logger.create () in - let%bind.Deferred (verify_blockchain_snarks, verify_transaction_snarks) = + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = instantiate_verify_functions ~logger config_file in let open Deferred.Let_syntax in let metadata_paths = get_filenames inputs in - let module V = Validator(Submission.JSON) in + let module V = Validator (Submission.JSON) in Deferred.List.iter metadata_paths ~f:(fun path -> match%bind - ( let open Deferred.Result.Let_syntax in - let%bind submission = - Submission.JSON.load ~block_dir path - |> Deferred.return - in - V.validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks submission) + let open Deferred.Result.Let_syntax in + let%bind submission = + Submission.JSON.load ~block_dir path |> Deferred.return + in + V.validate ~no_checks ~verify_blockchain_snarks + ~verify_transaction_snarks submission with | Ok (state_hash, parent, height, slot) -> display { state_hash; parent; height; slot } ; Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; - Deferred.unit)) + display_error @@ Error.to_string_hum e ; + Deferred.unit )) + +let cassandra_command = + Command.async ~summary:"Verify submissions and block read from Cassandra" + Command.Let_syntax.( + let%map_open cqlsh = cassandra_executable_flag + and _period_start = timestamp + and _period_end = timestamp in + fun () -> + let open Deferred.Let_syntax in + match%bind + Cassandra.exec ?cqlsh ~keyspace:"bpu_integration_dev" + @@ Cassandra.query + "SELECT JSON created_at, peer_id, snark_work, remote_addr, \ + submitter, block_hash, graphql_control_port FROM submissions;" + with + | Ok subs -> + List.iter subs + ~f: + (Async.printf "submission: '%a'\n" (fun () s -> + Submission.JSON.raw_to_yojson s + |> Yojson.Safe.pretty_to_string ) ) ; + Deferred.unit + | Error e -> + display_error @@ Error.to_string_hum e ; + Deferred.unit) + +let command = + Command.group + ~summary:"A tool for verifying JSON payload submitted by the uptime service" + [ ("fs", filesystem_command); ("cassandra", cassandra_command) ] let () = Async.Command.run command diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 09247d47f0e2..044220e642ca 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -82,4 +82,3 @@ module JSON = struct in { submitter; snark_work; block_hash = meta.block_hash; block_dir } end - From 41c96bb8850da6ec1d4b90e6844aa5f75893c6ec Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 14 Nov 2023 12:24:36 +0100 Subject: [PATCH 059/111] [delegation_verify] Refactor to abstract over data source. --- src/app/delegation_verify/cassandra.ml | 15 +- src/app/delegation_verify/cassandra.mli | 2 +- .../delegation_verify/delegation_verify.ml | 140 ++++++++++-------- src/app/delegation_verify/dune | 2 + src/app/delegation_verify/known_blocks.ml | 67 +++++++++ src/app/delegation_verify/submission.ml | 130 ++++++++-------- 6 files changed, 225 insertions(+), 131 deletions(-) create mode 100644 src/app/delegation_verify/known_blocks.ml diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 4acacae6a735..f51a1bed3de4 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -46,21 +46,26 @@ let read_json_line chan = | `Ok line -> ( try let j = Yojson.Safe.from_string line in - match Submission.JSON.raw_of_yojson j with + match Submission.raw_of_yojson j with | Ppx_deriving_yojson_runtime.Result.Ok s -> Ok s | Ppx_deriving_yojson_runtime.Result.Error e -> Or_error.error_string e with Yojson.Json_error e -> Or_error.error_string e ) +let skip_line chan = + let open Deferred.Let_syntax in + let%map _ = Reader.read_line chan in + Ok () + let rec read_json_output acc chan = let open Deferred.Or_error.Let_syntax in (* skip empty line *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in - (* [json] header *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + let%bind () = skip_line chan in (* [json] header *) - let%bind _ = Reader.read_line chan |> Deferred.map ~f:Or_error.return in + let%bind () = skip_line chan in + (* separator line *) + let%bind () = skip_line chan in match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with | Ok s -> read_json_output (s :: acc) chan diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index c1e30f7e977a..8ae44c0d024e 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -7,4 +7,4 @@ val lift : 'a Deferred.Or_error.t -> 'a t val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t -val query : string -> Submission.JSON.raw list t +val query : string -> Submission.raw list t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 8e41eab5c4aa..e5dcf404105f 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -10,52 +10,9 @@ let get_filenames = | filenames -> filenames -let verify_block ~verify_blockchain_snarks ~block = - let header = Mina_block.header block in - let open Mina_block.Header in - verify_blockchain_snarks - [ (protocol_state header, protocol_state_proof header) ] - let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] -module Validator (Sub : Submission.S) = struct - let validate ~no_checks ~verify_blockchain_snarks ~verify_transaction_snarks - submission = - let open Deferred.Result.Let_syntax in - let%bind block = Sub.block submission |> Deferred.return in - let%bind () = - if no_checks then return () - else verify_block ~verify_blockchain_snarks ~block - in - let%map () = - if no_checks then return () - else - match Sub.snark_work submission with - | None -> - Deferred.Result.return () - | Some - Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } - -> - let message = - Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:(Sub.submitter submission) - in - verify_snark_work ~verify_transaction_snarks ~proof ~message - in - let header = Mina_block.header block in - let protocol_state = Mina_block.Header.protocol_state header in - let consensus_state = - Mina_state.Protocol_state.consensus_state protocol_state - in - ( Mina_state.Protocol_state.hashes protocol_state - |> State_hash.State_hashes.state_hash - , Mina_state.Protocol_state.previous_state_hash protocol_state - , Consensus.Data.Consensus_state.blockchain_length consensus_state - , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state - ) -end - type valid_payload = { state_hash : State_hash.t ; parent : State_hash.t @@ -138,6 +95,66 @@ let instantiate_verify_functions ~logger = function in Verifier.verify_functions ~constraint_constants ~proof_level:Full () +module Make_verifier (Source : Submission.Data_source) = struct + let verify_transaction_snarks = Source.verify_transaction_snarks + + let verify_blockchain_snarks = Source.verify_blockchain_snarks + + let intialize_submission ?validate (src : Source.t) (sub : Submission.t) = + if Known_blocks.is_known sub.block_hash then () + else + Known_blocks.add ?validate ~verify_blockchain_snarks + ~block_hash:sub.block_hash + (Source.load_block src ~block_hash:sub.block_hash) + + let verify ~validate (submission : Submission.t) = + let open Deferred.Result.Let_syntax in + let%bind block = Known_blocks.get submission.block_hash in + let%bind () = Known_blocks.is_valid submission.block_hash in + let%map () = + if validate then + match submission.snark_work with + | None -> + Deferred.Result.return () + | Some + Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } + -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:submission.submitter + in + verify_snark_work ~verify_transaction_snarks ~proof ~message + else return () + in + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state + ) + + let validate_and_display_results ~validate submission = + let open Deferred.Let_syntax in + match%map verify ~validate submission with + | Ok (state_hash, parent, height, slot) -> + display { state_hash; parent; height; slot } + | Error e -> + display_error @@ Error.to_string_hum e + + let process ?(validate = true) (src : Source.t) = + let open Deferred.Or_error.Let_syntax in + let%bind submissions = Source.load_submissions src in + List.iter submissions ~f:(intialize_submission ~validate src) ; + List.map submissions ~f:(validate_and_display_results ~validate) + |> Deferred.all_unit + |> Deferred.map ~f:Or_error.return +end + let filesystem_command = Command.async ~summary:"Verify submissions and block read from the filesystem" Command.Let_syntax.( @@ -150,24 +167,23 @@ let filesystem_command = let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = instantiate_verify_functions ~logger config_file in + let submission_paths = get_filenames inputs in + let module V = Make_verifier (struct + include Submission.Filesystem + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in let open Deferred.Let_syntax in - let metadata_paths = get_filenames inputs in - let module V = Validator (Submission.JSON) in - Deferred.List.iter metadata_paths ~f:(fun path -> - match%bind - let open Deferred.Result.Let_syntax in - let%bind submission = - Submission.JSON.load ~block_dir path |> Deferred.return - in - V.validate ~no_checks ~verify_blockchain_snarks - ~verify_transaction_snarks submission - with - | Ok (state_hash, parent, height, slot) -> - display { state_hash; parent; height; slot } ; - Deferred.unit - | Error e -> - display_error @@ Error.to_string_hum e ; - Deferred.unit )) + match%bind + V.process ~validate:(not no_checks) { submission_paths; block_dir } + with + | Ok () -> + Deferred.unit + | Error e -> + display_error @@ Error.to_string_hum e ; + exit 1) let cassandra_command = Command.async ~summary:"Verify submissions and block read from Cassandra" @@ -187,8 +203,8 @@ let cassandra_command = List.iter subs ~f: (Async.printf "submission: '%a'\n" (fun () s -> - Submission.JSON.raw_to_yojson s - |> Yojson.Safe.pretty_to_string ) ) ; + Submission.raw_to_yojson s |> Yojson.Safe.pretty_to_string ) + ) ; Deferred.unit | Error e -> display_error @@ Error.to_string_hum e ; diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index bf8815b7b31c..ad78d6b827d1 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -14,6 +14,8 @@ base64 integers async.async_command + sexplib0 + sexplib ; mina libs signature_lib mina_block diff --git a/src/app/delegation_verify/known_blocks.ml b/src/app/delegation_verify/known_blocks.ml new file mode 100644 index 000000000000..92913e964167 --- /dev/null +++ b/src/app/delegation_verify/known_blocks.ml @@ -0,0 +1,67 @@ +open Async +open Core + +module Block_hash = struct + open Sexplib.Conv + + type t = string [@@deriving sexp, compare] + + let hash = Base.String.hash +end + +module Deferred_block = struct + type t = + { block : Mina_block.t Deferred.Or_error.t + ; valid : unit Deferred.Or_error.t (* Raises if block is invalid. *) + } + + let block_of_string contents = + let compute v = + let r = + try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) + with _ -> Error (Error.of_string "Fail to decode block") + in + Async.Ivar.fill v r + in + Deferred.create compute + + let verify_block ~verify_blockchain_snarks block = + let header = Mina_block.header block in + let open Mina_block.Header in + verify_blockchain_snarks + [ (protocol_state header, protocol_state_proof header) ] + + let create ?(validate = true) ~verify_blockchain_snarks contents = + let open Deferred.Or_error.Monad_infix in + let block = contents >>= block_of_string in + let valid = + if validate then block >>= verify_block ~verify_blockchain_snarks + else Deferred.Or_error.return () + in + { block; valid } +end + +let known_blocks : (Block_hash.t, Deferred_block.t) Hashtbl.t = + Hashtbl.create (module Block_hash) + +let add ?validate ~verify_blockchain_snarks ~block_hash block_contents = + let block = + Deferred_block.create ?validate ~verify_blockchain_snarks block_contents + in + Hashtbl.add_exn known_blocks ~key:block_hash ~data:block + +let is_known hash = Hashtbl.mem known_blocks hash + +let get hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.block + +let is_valid hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.valid diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 044220e642ca..5fbd4c6a7a71 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -1,52 +1,45 @@ +open Async open Core open Signature_lib -module type S = sig - type t - - val snark_work : t -> Uptime_service.Proof_data.t option +type t = + { snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + } - val submitter : t -> Public_key.Compressed.t +type submission = t - val block_hash : t -> string - - val block : t -> Mina_block.t Or_error.t -end +type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } +[@@deriving yojson] -module JSON = struct - type raw = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } - [@@deriving yojson] +module type Data_source = sig + type t - type t = - { snark_work : Uptime_service.Proof_data.t option - ; submitter : Public_key.Compressed.t - ; block_hash : string - ; block_dir : string - } + val load_submissions : t -> submission list Deferred.Or_error.t - let snark_work { snark_work; _ } = snark_work + val load_block : block_hash:string -> t -> string Deferred.Or_error.t - let submitter { submitter; _ } = submitter + val verify_blockchain_snarks : + (Mina_wire_types.Mina_state_protocol_state.Value.V2.t * Mina_base.Proof.t) + list + -> unit Async_kernel__Deferred_or_error.t - let block_hash { block_hash; _ } = block_hash + val verify_transaction_snarks : + (Ledger_proof.t * Mina_base.Sok_message.t) list + -> (unit, Error.t) Deferred.Result.t +end - let block { block_dir; block_hash; _ } = - let open Result.Let_syntax in - let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in - let%bind contents = - try Ok (In_channel.read_all block_path) - with _ -> Error (Error.of_string "Fail to load block") - in - try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) - with _ -> Error (Error.of_string "Fail to decode block") +module Filesystem = struct + type t = { block_dir : string; submission_paths : string list } let decode_snark_work str = match Base64.decode str with @@ -56,29 +49,40 @@ module JSON = struct | Error _ -> Error (Error.of_string "Fail to decode snark work") - let load ~block_dir filename = - let open Result.Let_syntax in - let%bind contents = - try Ok (In_channel.read_all filename) - with _ -> Error (Error.of_string "Fail to load metadata") - in - let%bind meta = - match Yojson.Safe.from_string contents |> raw_of_yojson with - | Ppx_deriving_yojson_runtime.Result.Ok a -> - Ok a - | Ppx_deriving_yojson_runtime.Result.Error e -> - Error (Error.of_string e) - in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter; snark_work; block_hash = meta.block_hash; block_dir } + let load_submissions { submission_paths; _ } = + Deferred.create (fun ivar -> + List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> + let open Result.Let_syntax in + let%bind acc = acc in + let%bind contents = + try Ok (In_channel.read_all filename) + with _ -> Error (Error.of_string "Fail to load metadata") + in + let%bind meta = + match Yojson.Safe.from_string contents |> raw_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (Error.of_string e) + in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash } :: acc ) + |> Ivar.fill ivar ) + + let load_block ~block_hash { block_dir; _ } = + Deferred.create (fun ivar -> + let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in + ( try Ok (In_channel.read_all block_path) + with _ -> Error (Error.of_string "Fail to load block") ) + |> Ivar.fill ivar ) end From e257f016ae32b2b2f25919be388c8882103ba9ec Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 15 Nov 2023 14:09:17 +0100 Subject: [PATCH 060/111] [delegation_verify] Load input from Cassandra. --- src/app/delegation_verify/cassandra.ml | 102 +++++------------- src/app/delegation_verify/cassandra.mli | 17 +-- .../delegation_verify/delegation_verify.ml | 41 ++++--- src/app/delegation_verify/dune | 1 + src/app/delegation_verify/submission.ml | 99 +++++++++++++---- 5 files changed, 142 insertions(+), 118 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index f51a1bed3de4..dcd57762688f 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -1,86 +1,34 @@ open Async open Core -type connection = Process.t +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -type 'a t = connection -> 'a Or_error.t Deferred.t - -module M = struct - type nonrec 'a t = 'a t - - let return x _ = return (Ok x) - - let map = `Custom (fun m ~f c -> m c >>| Or_error.map ~f) - - let bind m ~f c = - m c >>= function Error _ as e -> Deferred.return e | Ok x -> f x c -end - -include Monad.Make (M) - -let lift m _ = m - -(* Open connection to a Cassandra database using cqlsh executable - provided. The connection should be configured via ~/.cassandra/cqlshrc - configuration file. Return the process handle which can the be used to - send queries to the database. *) -let connect ?executable keyspace = +let query ?executable ~parse q = let open Deferred.Or_error.Let_syntax in let prog = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in - let%map proc = Process.create ~prog ~args:[ "-k"; keyspace ] () in - Async.printf "Connected to the Cassandra database\n" ; - proc - -let exec ?cqlsh ~keyspace m = - let open Deferred.Or_error.Monad_infix in - connect ?executable:cqlsh keyspace >>= m - -let read_json_line chan = - let open Deferred.Let_syntax in - match%map Reader.read_line chan with - | `Eof -> - Or_error.error_string "Cassandra client: broken pipe" - | `Ok line -> ( - try - let j = Yojson.Safe.from_string line in - match Submission.raw_of_yojson j with - | Ppx_deriving_yojson_runtime.Result.Ok s -> - Ok s - | Ppx_deriving_yojson_runtime.Result.Error e -> - Or_error.error_string e - with Yojson.Json_error e -> Or_error.error_string e ) - -let skip_line chan = - let open Deferred.Let_syntax in - let%map _ = Reader.read_line chan in - Ok () - -let rec read_json_output acc chan = - let open Deferred.Or_error.Let_syntax in - (* skip empty line *) - let%bind () = skip_line chan in - (* [json] header *) - let%bind () = skip_line chan in - (* separator line *) - let%bind () = skip_line chan in - match%bind read_json_line chan |> Deferred.map ~f:Or_error.return with - | Ok s -> - read_json_output (s :: acc) chan - | Error _ -> - return (List.rev acc) - -let query q conn = - let open Deferred.Let_syntax in - let proc_stdin = Process.stdin conn in - Writer.write_line proc_stdin q ; - Writer.(write_line @@ Lazy.force stdout) q ; - let%bind () = Writer.close proc_stdin in - let%bind () = Writer.flushed proc_stdin in - match%bind read_json_output [] (Process.stdout conn) with - | Ok output -> - return (Ok output) - | Error e -> - return (Error e) + printf "SQL: '%s'\n" q ; + let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in + List.slice data 3 (-2) (* skip header and footer *) + |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> + let open Or_error.Let_syntax in + let%bind l = acc in + try + let j = Yojson.Safe.from_string line in + match parse j with + | Ppx_deriving_yojson_runtime.Result.Ok s -> + Ok (s :: l) + | Ppx_deriving_yojson_runtime.Result.Error e -> + Or_error.error_string e + with Yojson.Json_error e -> Or_error.error_string e ) + |> Deferred.return + +let select ?executable ~keyspace ~parse ~fields ?where from = + query ?executable ~parse + @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" + (String.concat ~sep:"," fields) + keyspace + from + (match where with None -> "" | Some w -> " WHERE " ^ w) diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index 8ae44c0d024e..0a3a67337c7c 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -1,10 +1,15 @@ open Async -open Core -include Monad.S +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -val lift : 'a Deferred.Or_error.t -> 'a t +val query : + ?executable:string -> parse:'a parser -> string -> 'a list Deferred.Or_error.t -val exec : ?cqlsh:string -> keyspace:string -> 'a t -> 'a Deferred.Or_error.t - -val query : string -> Submission.raw list t +val select : + ?executable:string + -> keyspace:string + -> parse:'a parser + -> fields:string list + -> ?where:string + -> string + -> 'a list Deferred.Or_error.t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index e5dcf404105f..c5c42700fe04 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -189,26 +189,37 @@ let cassandra_command = Command.async ~summary:"Verify submissions and block read from Cassandra" Command.Let_syntax.( let%map_open cqlsh = cassandra_executable_flag - and _period_start = timestamp - and _period_end = timestamp in + and no_checks = no_checks_flag + and config_file = config_flag + and period_start = timestamp + and period_end = timestamp in fun () -> let open Deferred.Let_syntax in - match%bind - Cassandra.exec ?cqlsh ~keyspace:"bpu_integration_dev" - @@ Cassandra.query - "SELECT JSON created_at, peer_id, snark_work, remote_addr, \ - submitter, block_hash, graphql_control_port FROM submissions;" - with - | Ok subs -> - List.iter subs - ~f: - (Async.printf "submission: '%a'\n" (fun () s -> - Submission.raw_to_yojson s |> Yojson.Safe.pretty_to_string ) - ) ; + let logger = Logger.create () in + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = + instantiate_verify_functions ~logger config_file + in + let module V = Make_verifier (struct + include Submission.Cassandra + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in + let src = + Submission.Cassandra. + { executable = cqlsh + ; keyspace = "bpu_integration_dev" + ; period_start + ; period_end + } + in + match%bind V.process ~validate:(not no_checks) src with + | Ok () -> Deferred.unit | Error e -> display_error @@ Error.to_string_hum e ; - Deferred.unit) + exit 1) let command = Command.group diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index ad78d6b827d1..7c0f031cc18e 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -9,6 +9,7 @@ stdio base base.caml + hex ppx_deriving_yojson.runtime yojson base64 diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5fbd4c6a7a71..d860cb8a7a9c 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -21,6 +21,29 @@ type raw = } [@@deriving yojson] +let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error (Error.of_string "Fail to decode snark work") ) + | Error _ -> + Error (Error.of_string "Fail to decode snark work") + +let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter; snark_work; block_hash = meta.block_hash } + module type Data_source = sig type t @@ -41,14 +64,6 @@ end module Filesystem = struct type t = { block_dir : string; submission_paths : string list } - let decode_snark_work str = - match Base64.decode str with - | Ok str -> ( - try Ok (Binable.of_string (module Uptime_service.Proof_data) str) - with _ -> Error (Error.of_string "Fail to decode snark work") ) - | Error _ -> - Error (Error.of_string "Fail to decode snark work") - let load_submissions { submission_paths; _ } = Deferred.create (fun ivar -> List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> @@ -65,18 +80,8 @@ module Filesystem = struct | Ppx_deriving_yojson_runtime.Result.Error e -> Error (Error.of_string e) in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter; snark_work; block_hash = meta.block_hash } :: acc ) + let%map t = of_raw meta in + t :: acc ) |> Ivar.fill ivar ) let load_block ~block_hash { block_dir; _ } = @@ -86,3 +91,57 @@ module Filesystem = struct with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) end + +module Cassandra = struct + type t = + { executable : string option + ; keyspace : string + ; period_start : string + ; period_end : string + } + + type block_data = { raw_block : string } [@@deriving yojson] + + let load_submissions { executable; keyspace; period_start; period_end } = + let open Deferred.Or_error.Let_syntax in + let%bind raw = + Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace + ~fields: + [ "created_at" + ; "peer_id" + ; "snark_work" + ; "remote_addr" + ; "submitter" + ; "block_hash" + ; "graphql_control_port" + ] + ~where: + (sprintf + "submitted_at_date = '%s' AND submitted_at >= '%s' AND \ + submitted_at <= '%s'" + (List.hd_exn @@ String.split ~on:' ' period_start) + period_start period_end ) + "submissions" + in + List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> + let open Result.Let_syntax in + let%bind l = acc in + printf "sub: '%s'\n" (Yojson.Safe.to_string @@ raw_to_yojson sub) ; + let%map s = of_raw sub in + s :: l ) + |> Deferred.return + + let load_block ~block_hash { executable; keyspace; _ } = + let open Deferred.Or_error.Let_syntax in + let%bind block_data = + Cassandra.select ?executable ~parse:block_data_of_yojson ~keyspace + ~fields:[ "raw_block" ] + ~where:(sprintf "block_hash = '%s'" block_hash) + "blocks" + in + match List.hd block_data with + | None -> + Deferred.Or_error.error_string "Cassandra: Block not found" + | Some b -> + Hex.Safe.of_hex b.raw_block |> Option.value_exn |> return +end From 06f2bf0655c8e9bcb8220c59e9a1dbb1b2c70919 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 10:36:13 +0100 Subject: [PATCH 061/111] Skip empty lines in order to avoid JSON parsing errors. An empty line sneaks in in particular when Casandra finds no results. --- src/app/delegation_verify/cassandra.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index dcd57762688f..b487bc06e07e 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -12,6 +12,7 @@ let query ?executable ~parse q = printf "SQL: '%s'\n" q ; let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in List.slice data 3 (-2) (* skip header and footer *) + |> List.filter ~f:(fun s -> not (String.is_empty s)) |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> let open Or_error.Let_syntax in let%bind l = acc in From 02f85b1b55b599988f447b5b9efc59d636f9efe6 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 11:38:50 +0100 Subject: [PATCH 062/111] [delegation_verify] Fix problems with reading Cassandra blobs. --- src/app/delegation_verify/cassandra.ml | 3 +-- src/app/delegation_verify/dune | 2 +- src/app/delegation_verify/submission.ml | 11 ++++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index b487bc06e07e..5630f3a758a1 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -30,6 +30,5 @@ let select ?executable ~keyspace ~parse ~fields ?where from = query ?executable ~parse @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" (String.concat ~sep:"," fields) - keyspace - from + keyspace from (match where with None -> "" | Some w -> " WHERE " ^ w) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index 7c0f031cc18e..3de6f7938743 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -9,7 +9,6 @@ stdio base base.caml - hex ppx_deriving_yojson.runtime yojson base64 @@ -17,6 +16,7 @@ async.async_command sexplib0 sexplib + hex ; mina libs signature_lib mina_block diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index d860cb8a7a9c..0e0dff8ae6d5 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -126,8 +126,12 @@ module Cassandra = struct List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> let open Result.Let_syntax in let%bind l = acc in - printf "sub: '%s'\n" (Yojson.Safe.to_string @@ raw_to_yojson sub) ; - let%map s = of_raw sub in + let snark_work = + Option.map sub.snark_work ~f:(fun s -> + String.chop_prefix_exn s ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> Base64.encode_string ) + in + let%map s = of_raw { sub with snark_work } in s :: l ) |> Deferred.return @@ -143,5 +147,6 @@ module Cassandra = struct | None -> Deferred.Or_error.error_string "Cassandra: Block not found" | Some b -> - Hex.Safe.of_hex b.raw_block |> Option.value_exn |> return + String.chop_prefix_exn b.raw_block ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> return end From 14801eac10b1fcc143135865a671dba8d5f4787a Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 14:04:27 +0100 Subject: [PATCH 063/111] [delegation_verify] Add identification data to the output. --- .../delegation_verify/delegation_verify.ml | 28 +++++++++++++------ src/app/delegation_verify/submission.ml | 9 ++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index c5c42700fe04..06f5f11b72b0 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -14,19 +14,22 @@ let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] type valid_payload = - { state_hash : State_hash.t + { created_at : string + ; submitter : string + ; state_hash : State_hash.t ; parent : State_hash.t ; height : Unsigned.uint32 ; slot : Mina_numbers.Global_slot_since_genesis.t } -let valid_payload_to_yojson { state_hash; parent; height; slot } : Yojson.Safe.t - = +let valid_payload_to_yojson (p : valid_payload) : Yojson.Safe.t = `Assoc - [ ("state_hash", State_hash.to_yojson state_hash) - ; ("parent", State_hash.to_yojson parent) - ; ("height", `Int (Unsigned.UInt32.to_int height)) - ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int slot)) + [ ("created_at", `String p.created_at) + ; ("submitter", `String p.submitter) + ; ("state_hash", State_hash.to_yojson p.state_hash) + ; ("parent", State_hash.to_yojson p.parent) + ; ("height", `Int (Unsigned.UInt32.to_int p.height)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) ] let display valid_payload = @@ -142,7 +145,16 @@ module Make_verifier (Source : Submission.Data_source) = struct let open Deferred.Let_syntax in match%map verify ~validate submission with | Ok (state_hash, parent, height, slot) -> - display { state_hash; parent; height; slot } + display + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + } | Error e -> display_error @@ Error.to_string_hum e diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 0e0dff8ae6d5..5baebda17822 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -3,7 +3,8 @@ open Core open Signature_lib type t = - { snark_work : Uptime_service.Proof_data.t option + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option ; submitter : Public_key.Compressed.t ; block_hash : string } @@ -42,7 +43,11 @@ let of_raw meta = let%map snark_work = decode_snark_work s in Some snark_work in - { submitter; snark_work; block_hash = meta.block_hash } + { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + } module type Data_source = sig type t From 0b6c008b46e21689a0598b91b35a05008efdeb23 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 15:45:37 +0100 Subject: [PATCH 064/111] [delegation_verify] Abstract over output mechanism. In order to allow for outputting results to Cassandra. --- src/app/delegation_verify/cassandra.ml | 32 ++++++--- src/app/delegation_verify/cassandra.mli | 11 +++- .../delegation_verify/delegation_verify.ml | 66 ++++++------------- src/app/delegation_verify/output.ml | 28 ++++++++ src/app/delegation_verify/submission.ml | 13 ++++ 5 files changed, 91 insertions(+), 59 deletions(-) create mode 100644 src/app/delegation_verify/output.ml diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 5630f3a758a1..db6e2c1bfc7a 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -3,14 +3,23 @@ open Core type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -let query ?executable ~parse q = - let open Deferred.Or_error.Let_syntax in +let query ?executable q = let prog = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in printf "SQL: '%s'\n" q ; - let%bind data = Process.run_lines ~prog ~stdin:q ~args:[] () in + Process.run_lines ~prog ~stdin:q ~args:[] () + +let select ?executable ~keyspace ~parse ~fields ?where from = + let open Deferred.Or_error.Let_syntax in + let%bind data = + query ?executable + @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" + (String.concat ~sep:"," fields) + keyspace from + (match where with None -> "" | Some w -> " WHERE " ^ w) + in List.slice data 3 (-2) (* skip header and footer *) |> List.filter ~f:(fun s -> not (String.is_empty s)) |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> @@ -26,9 +35,14 @@ let query ?executable ~parse q = with Yojson.Json_error e -> Or_error.error_string e ) |> Deferred.return -let select ?executable ~keyspace ~parse ~fields ?where from = - query ?executable ~parse - @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" - (String.concat ~sep:"," fields) - keyspace from - (match where with None -> "" | Some w -> " WHERE " ^ w) +let update ?executable ~keyspace ~table ~where updates = + let open Deferred.Or_error.Let_syntax in + let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in + let%map _ = + query ?executable + @@ Printf.sprintf "UPDATE %s.%s SET %s WHERE %s;" + keyspace table + (String.concat ~sep:"," assignments) + where + in + () diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index 0a3a67337c7c..990bd0de6b90 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -2,9 +2,6 @@ open Async type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -val query : - ?executable:string -> parse:'a parser -> string -> 'a list Deferred.Or_error.t - val select : ?executable:string -> keyspace:string @@ -13,3 +10,11 @@ val select : -> ?where:string -> string -> 'a list Deferred.Or_error.t + +val update : + ?executable:string + -> keyspace:string + -> table:string + -> where:string + -> (string * string) list + -> unit Deferred.Or_error.t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 06f5f11b72b0..3d534dd5980c 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -13,32 +13,6 @@ let get_filenames = let verify_snark_work ~verify_transaction_snarks ~proof ~message = verify_transaction_snarks [ (proof, message) ] -type valid_payload = - { created_at : string - ; submitter : string - ; state_hash : State_hash.t - ; parent : State_hash.t - ; height : Unsigned.uint32 - ; slot : Mina_numbers.Global_slot_since_genesis.t - } - -let valid_payload_to_yojson (p : valid_payload) : Yojson.Safe.t = - `Assoc - [ ("created_at", `String p.created_at) - ; ("submitter", `String p.submitter) - ; ("state_hash", State_hash.to_yojson p.state_hash) - ; ("parent", State_hash.to_yojson p.parent) - ; ("height", `Int (Unsigned.UInt32.to_int p.height)) - ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) - ] - -let display valid_payload = - printf "%s\n" @@ Yojson.Safe.to_string - @@ valid_payload_to_yojson valid_payload - -let display_error e = - eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] - let config_flag = let open Command.Param in flag "--config-file" ~doc:"FILE config file" (optional string) @@ -90,7 +64,7 @@ let instantiate_verify_functions ~logger = function | Ok (precomputed_values, _) -> Deferred.return precomputed_values | Error _ -> - display_error "fail to read config file" ; + Output.display_error "fail to read config file" ; exit 4 in let constraint_constants = @@ -141,30 +115,28 @@ module Make_verifier (Source : Submission.Data_source) = struct , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state ) - let validate_and_display_results ~validate submission = + let validate_and_display_results ~validate ~src submission = let open Deferred.Let_syntax in - match%map verify ~validate submission with - | Ok (state_hash, parent, height, slot) -> - display - { created_at = submission.created_at - ; submitter = - Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter - ; state_hash - ; parent - ; height - ; slot - } - | Error e -> - display_error @@ Error.to_string_hum e + let%bind result = verify ~validate submission in + Result.map result ~f:(fun (state_hash, parent, height, slot) -> + Output. + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + }) + |> Source.output src submission let process ?(validate = true) (src : Source.t) = let open Deferred.Or_error.Let_syntax in let%bind submissions = Source.load_submissions src in List.iter submissions ~f:(intialize_submission ~validate src) ; - List.map submissions ~f:(validate_and_display_results ~validate) - |> Deferred.all_unit - |> Deferred.map ~f:Or_error.return + List.map submissions ~f:(validate_and_display_results ~src ~validate) + |> Deferred.Or_error.all_unit end let filesystem_command = @@ -194,7 +166,7 @@ let filesystem_command = | Ok () -> Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; + Output.display_error @@ Error.to_string_hum e ; exit 1) let cassandra_command = @@ -230,7 +202,7 @@ let cassandra_command = | Ok () -> Deferred.unit | Error e -> - display_error @@ Error.to_string_hum e ; + Output.display_error @@ Error.to_string_hum e ; exit 1) let command = diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml new file mode 100644 index 000000000000..952f26140f84 --- /dev/null +++ b/src/app/delegation_verify/output.ml @@ -0,0 +1,28 @@ +open Async +open Mina_base + +type t = + { created_at : string + ; submitter : string + ; state_hash : State_hash.t + ; parent : State_hash.t + ; height : Unsigned.uint32 + ; slot : Mina_numbers.Global_slot_since_genesis.t + } + +let valid_payload_to_yojson (p : t) : Yojson.Safe.t = + `Assoc + [ ("created_at", `String p.created_at) + ; ("submitter", `String p.submitter) + ; ("state_hash", State_hash.to_yojson p.state_hash) + ; ("parent", State_hash.to_yojson p.parent) + ; ("height", `Int (Unsigned.UInt32.to_int p.height)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) + ] + +let display valid_payload = + printf "%s\n" @@ Yojson.Safe.to_string + @@ valid_payload_to_yojson valid_payload + +let display_error e = + eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5baebda17822..e6e8e9f9b063 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -64,6 +64,8 @@ module type Data_source = sig val verify_transaction_snarks : (Ledger_proof.t * Mina_base.Sok_message.t) list -> (unit, Error.t) Deferred.Result.t + + val output : t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t end module Filesystem = struct @@ -95,6 +97,15 @@ module Filesystem = struct ( try Ok (In_channel.read_all block_path) with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) + + let output _ (_submission : submission) = function + | Ok payload -> + Output.display payload ; + Deferred.Or_error.return () + + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Deferred.Or_error.return () end module Cassandra = struct @@ -154,4 +165,6 @@ module Cassandra = struct | Some b -> String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return + + let output = Filesystem.output end From 7b3baab1bdd7fc94aa5c97abdcfa66b50294b13e Mon Sep 17 00:00:00 2001 From: Sventimir Date: Thu, 16 Nov 2023 16:25:14 +0100 Subject: [PATCH 065/111] [delegation_verify] Upload results to Cassandra. --- src/app/delegation_verify/cassandra.ml | 6 ++-- .../delegation_verify/delegation_verify.ml | 20 +++++------ src/app/delegation_verify/output.ml | 8 +++++ src/app/delegation_verify/submission.ml | 34 ++++++++++++++++--- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index db6e2c1bfc7a..56a2c378af2f 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -13,7 +13,7 @@ let query ?executable q = let select ?executable ~keyspace ~parse ~fields ?where from = let open Deferred.Or_error.Let_syntax in - let%bind data = + let%bind data = query ?executable @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" (String.concat ~sep:"," fields) @@ -38,9 +38,9 @@ let select ?executable ~keyspace ~parse ~fields ?where from = let update ?executable ~keyspace ~table ~where updates = let open Deferred.Or_error.Let_syntax in let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in - let%map _ = + let%map _ = query ?executable - @@ Printf.sprintf "UPDATE %s.%s SET %s WHERE %s;" + @@ Printf.sprintf "CONSISTENCY LOCAL_QUORUM; UPDATE %s.%s SET %s WHERE %s;" keyspace table (String.concat ~sep:"," assignments) where diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 3d534dd5980c..6a4cbfc423ca 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -119,16 +119,16 @@ module Make_verifier (Source : Submission.Data_source) = struct let open Deferred.Let_syntax in let%bind result = verify ~validate submission in Result.map result ~f:(fun (state_hash, parent, height, slot) -> - Output. - { created_at = submission.created_at - ; submitter = - Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter - ; state_hash - ; parent - ; height - ; slot - }) + Output. + { created_at = submission.created_at + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + submission.submitter + ; state_hash + ; parent + ; height + ; slot + } ) |> Source.output src submission let process ?(validate = true) (src : Source.t) = diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml index 952f26140f84..1d98d1dd55f4 100644 --- a/src/app/delegation_verify/output.ml +++ b/src/app/delegation_verify/output.ml @@ -20,6 +20,14 @@ let valid_payload_to_yojson (p : t) : Yojson.Safe.t = ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) ] +let valid_payload_to_cassandra_updates (p : t) = + [ ("height", Unsigned.UInt32.to_string p.height) + ; ("slot", Mina_numbers.Global_slot_since_genesis.to_string p.slot) + ; ("parent", Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.parent) + ; ( "state_hash" + , Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.state_hash ) + ] + let display valid_payload = printf "%s\n" @@ Yojson.Safe.to_string @@ valid_payload_to_yojson valid_payload diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index e6e8e9f9b063..a3b34c523d84 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -65,7 +65,8 @@ module type Data_source = sig (Ledger_proof.t * Mina_base.Sok_message.t) list -> (unit, Error.t) Deferred.Result.t - val output : t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t + val output : + t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t end module Filesystem = struct @@ -100,9 +101,8 @@ module Filesystem = struct let output _ (_submission : submission) = function | Ok payload -> - Output.display payload ; - Deferred.Or_error.return () - + Output.display payload ; + Deferred.Or_error.return () | Error e -> Output.display_error @@ Error.to_string_hum e ; Deferred.Or_error.return () @@ -166,5 +166,29 @@ module Cassandra = struct String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return - let output = Filesystem.output + let output src (submission : submission) = function + | Ok payload -> + Output.display payload ; + Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.created_at) + submission.created_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + Output.(valid_payload_to_cassandra_updates payload) + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.created_at) + submission.created_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) ] end From 7f5ab0547129bca453acefe6eb70dbdb9f44b5a3 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 17 Nov 2023 14:44:42 +0100 Subject: [PATCH 066/111] [delegation_verify] Use the proper keys for Cassandra records. --- .../delegation_verify/delegation_verify.ml | 26 +-- src/app/delegation_verify/submission.ml | 148 +++++++++++++----- 2 files changed, 123 insertions(+), 51 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 6a4cbfc423ca..e7cbfffd8372 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -77,20 +77,22 @@ module Make_verifier (Source : Submission.Data_source) = struct let verify_blockchain_snarks = Source.verify_blockchain_snarks - let intialize_submission ?validate (src : Source.t) (sub : Submission.t) = - if Known_blocks.is_known sub.block_hash then () + let intialize_submission ?validate (src : Source.t) (sub : Source.submission) + = + let block_hash = Source.block_hash sub in + if Known_blocks.is_known block_hash then () else - Known_blocks.add ?validate ~verify_blockchain_snarks - ~block_hash:sub.block_hash - (Source.load_block src ~block_hash:sub.block_hash) + Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash + (Source.load_block src ~block_hash) - let verify ~validate (submission : Submission.t) = + let verify ~validate (submission : Source.submission) = let open Deferred.Result.Let_syntax in - let%bind block = Known_blocks.get submission.block_hash in - let%bind () = Known_blocks.is_valid submission.block_hash in + let block_hash = Source.block_hash submission in + let%bind block = Known_blocks.get block_hash in + let%bind () = Known_blocks.is_valid block_hash in let%map () = if validate then - match submission.snark_work with + match Source.snark_work submission with | None -> Deferred.Result.return () | Some @@ -98,7 +100,7 @@ module Make_verifier (Source : Submission.Data_source) = struct -> let message = Mina_base.Sok_message.create ~fee:snark_work_fee - ~prover:submission.submitter + ~prover:(Source.submitter submission) in verify_snark_work ~verify_transaction_snarks ~proof ~message else return () @@ -120,10 +122,10 @@ module Make_verifier (Source : Submission.Data_source) = struct let%bind result = verify ~validate submission in Result.map result ~f:(fun (state_hash, parent, height, slot) -> Output. - { created_at = submission.created_at + { created_at = Source.created_at submission ; submitter = Signature_lib.Public_key.Compressed.to_base58_check - submission.submitter + (Source.submitter submission) ; state_hash ; parent ; height diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index a3b34c523d84..308bbca441ff 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -2,26 +2,6 @@ open Async open Core open Signature_lib -type t = - { created_at : string - ; snark_work : Uptime_service.Proof_data.t option - ; submitter : Public_key.Compressed.t - ; block_hash : string - } - -type submission = t - -type raw = - { created_at : string - ; peer_id : string - ; snark_work : string option [@default None] - ; remote_addr : string - ; submitter : string - ; block_hash : string - ; graphql_control_port : int option [@default None] - } -[@@deriving yojson] - let decode_snark_work str = match Base64.decode str with | Ok str -> ( @@ -30,28 +10,19 @@ let decode_snark_work str = | Error _ -> Error (Error.of_string "Fail to decode snark work") -let of_raw meta = - let open Result.Let_syntax in - let%bind.Result submitter = - Public_key.Compressed.of_base58_check meta.submitter - in - let%map snark_work = - match meta.snark_work with - | None -> - Ok None - | Some s -> - let%map snark_work = decode_snark_work s in - Some snark_work - in - { submitter - ; snark_work - ; block_hash = meta.block_hash - ; created_at = meta.created_at - } - module type Data_source = sig type t + type submission + + val created_at : submission -> string + + val block_hash : submission -> string + + val snark_work : submission -> Uptime_service.Proof_data.t option + + val submitter : submission -> Public_key.Compressed.t + val load_submissions : t -> submission list Deferred.Or_error.t val load_block : block_hash:string -> t -> string Deferred.Or_error.t @@ -72,6 +43,51 @@ end module Filesystem = struct type t = { block_dir : string; submission_paths : string list } + type submission = + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + } + + let created_at ({ created_at; _ } : submission) = created_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + let load_submissions { submission_paths; _ } = Deferred.create (fun ivar -> List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> @@ -118,12 +134,66 @@ module Cassandra = struct type block_data = { raw_block : string } [@@deriving yojson] + type submission = + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + ; submitted_at : string + ; submitted_at_date : string + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + ; submitted_at : string + ; submitted_at_date : string + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + ( { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + ; submitted_at_date = meta.submitted_at_date + ; submitted_at = meta.submitted_at + } + : submission ) + + let created_at ({ created_at; _ } : submission) = created_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in let%bind raw = Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace ~fields: [ "created_at" + ; "submitted_at_date" + ; "submitted_at" ; "peer_id" ; "snark_work" ; "remote_addr" From c5a2586587ab28f29009cc38a10c793c6b4adca2 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Fri, 17 Nov 2023 15:32:07 +0100 Subject: [PATCH 067/111] [delegation_verify] Allow for querying entries from multiple days. --- src/app/delegation_verify/submission.ml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 308bbca441ff..67b4162053cc 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -188,6 +188,20 @@ module Cassandra = struct let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in + let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in + let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in + let partition_keys = + Date.dates_between ~min:start_day ~max:end_day + |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") + in + let partition = + if List.length partition_keys = 1 then + sprintf "submitted_at_date = '%s'" (List.hd_exn partition_keys) + else + sprintf + "submitted_at_date IN (%s)" + (String.concat ~sep:"," @@ List.map ~f:(sprintf "'%s'") partition_keys) + in let%bind raw = Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace ~fields: @@ -203,9 +217,8 @@ module Cassandra = struct ] ~where: (sprintf - "submitted_at_date = '%s' AND submitted_at >= '%s' AND \ - submitted_at <= '%s'" - (List.hd_exn @@ String.split ~on:' ' period_start) + "%s AND submitted_at >= '%s' AND submitted_at <= '%s'" + partition period_start period_end ) "submissions" in From 6eaa437eab615221da1a0d5bc165af3fc2d83267 Mon Sep 17 00:00:00 2001 From: Smorci Date: Fri, 17 Nov 2023 17:14:53 +0000 Subject: [PATCH 068/111] PM-633 Add delegation_verify package and docker image, update libp2p hash --- flake.nix | 4 ++-- nix/docker.nix | 10 ++++++++++ nix/libp2p_helper.json | 2 +- nix/ocaml.nix | 24 +++++++++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 1bae798d9b02..0136412c2158 100644 --- a/flake.nix +++ b/flake.nix @@ -287,12 +287,12 @@ # Main user-facing binaries. packages = rec { inherit (ocamlPackages) - mina mina_tests mina-ocaml-format test_executive; + mina mina_tests mina-ocaml-format test_executive mina-delegation-verify; inherit (pkgs) libp2p_helper kimchi_bindings_stubs snarky_js leaderboard validation trace-tool zkapp-cli; inherit (dockerImages) - mina-image-slim mina-image-full mina-archive-image-full; + mina-image-slim mina-image-full mina-archive-image-full mina-delegation-verify-image; mina-deb = debianPackages.mina; default = mina; }; diff --git a/nix/docker.nix b/nix/docker.nix index 748c23369522..3655dc2fd37e 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -77,6 +77,16 @@ in { inherit created; contents = [ ocamlPackages_mina.mina.out ]; }; + + mina-delegation-verify-image = dockerTools.streamLayeredImage { + name = "mina-delegation-verify"; + inherit created; + contents = [ ocamlPackages_mina.mina-delegation-verify.out ]; + config = { + cmd = [ "/bin/delegation-verify" ]; + }; + }; + mina-image-full = mkFullImage "mina" (with ocamlPackages_mina; [ mina-build-config mina-daemon-scripts diff --git a/nix/libp2p_helper.json b/nix/libp2p_helper.json index 7a0a9b7e34a7..650889eddc1c 100644 --- a/nix/libp2p_helper.json +++ b/nix/libp2p_helper.json @@ -1 +1 @@ -{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} \ No newline at end of file +{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} diff --git a/nix/ocaml.nix b/nix/ocaml.nix index d6a0be0a9e88..defb546007a9 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -98,7 +98,8 @@ let --set MINA_LIBP2P_HELPER_PATH ${pkgs.libp2p_helper}/bin/libp2p_helper \ --set MINA_COMMIT_SHA1 ${escapeShellArg commit_sha1} \ --set MINA_COMMIT_DATE ${escapeShellArg commit_date} \ - --set MINA_BRANCH "''${MINA_BRANCH-}" + --set MINA_BRANCH "''${MINA_BRANCH-}" \ + --set TZDIR "${pkgs.tzdata}/share/zoneinfo" done '') package.outputs); @@ -321,6 +322,27 @@ let ''; }); + # Stateless verification tool + mina-delegation-verify-dev = self.mina-dev.overrideAttrs (oa: { + pname = "mina-delegation-verify"; + version = "dev"; + outputs = [ "out" ]; + + buildInputs = oa.buildInputs ++ [ pkgs.cassandra_4 ]; + + buildPhase = '' + dune build --display=short \ + src/app/delegation_verify/delegation_verify.exe + ''; + + installPhase = '' + mkdir -p $out/bin + mv _build/default/src/app/delegation_verify/delegation_verify.exe $out/bin/delegation-verify + ''; + }); + + mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; + # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { pname = "mina-test_executive"; From eabbca386d0a6e48ee8489d9003b1dd936e36440 Mon Sep 17 00:00:00 2001 From: Smorci Date: Mon, 20 Nov 2023 10:34:17 +0000 Subject: [PATCH 069/111] PM-633 Add debugging tools --- nix/docker.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docker.nix b/nix/docker.nix index 3655dc2fd37e..61a67b194402 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -81,7 +81,7 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; - contents = [ ocamlPackages_mina.mina-delegation-verify.out ]; + contents = [ ocamlPackages_mina.mina-delegation-verify.out coreutils bashInteractive ]; config = { cmd = [ "/bin/delegation-verify" ]; }; From e452666faa37e7f5a6aca218fa5594677b85e8bb Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 20 Nov 2023 14:11:56 +0100 Subject: [PATCH 070/111] [delegation_verify] Don't use core's dates_between function. Dates.dates_between tries to access /etc/localtime, which doesn't seem right. --- src/app/delegation_verify/cassandra.ml | 1 - src/app/delegation_verify/submission.ml | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 56a2c378af2f..8b906e6bf274 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -8,7 +8,6 @@ let query ?executable q = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in - printf "SQL: '%s'\n" q ; Process.run_lines ~prog ~stdin:q ~args:[] () let select ?executable ~keyspace ~parse ~fields ?where from = diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 67b4162053cc..fc89ab8a60ae 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -186,12 +186,19 @@ module Cassandra = struct let submitter ({ submitter; _ } : submission) = submitter + let dates_between start finish = + let rec go acc current = + if Date.(current > finish) then acc + else go (current :: acc) Date.(add_days current 1) + in + go [] start + let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in let partition_keys = - Date.dates_between ~min:start_day ~max:end_day + dates_between start_day end_day |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") in let partition = From aef3a08a220b78e45293650d1465744ae99851f1 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 20 Nov 2023 14:41:09 +0100 Subject: [PATCH 071/111] [delegation_verify] Make Cassandra keyspace parametric. --- src/app/delegation_verify/delegation_verify.ml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index e7cbfffd8372..d478bf8a385d 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -17,6 +17,10 @@ let config_flag = let open Command.Param in flag "--config-file" ~doc:"FILE config file" (optional string) +let keyspace_flag = + let open Command.Param in + flag "--keyspace" ~doc:"Name of the Cassandra keyspace" (required string) + let no_checks_flag = let open Command.Param in flag "--no-checks" ~aliases:[ "-no-checks" ] @@ -177,6 +181,7 @@ let cassandra_command = let%map_open cqlsh = cassandra_executable_flag and no_checks = no_checks_flag and config_file = config_flag + and keyspace = keyspace_flag and period_start = timestamp and period_end = timestamp in fun () -> @@ -195,7 +200,7 @@ let cassandra_command = let src = Submission.Cassandra. { executable = cqlsh - ; keyspace = "bpu_integration_dev" + ; keyspace ; period_start ; period_end } From e110c2b974db1c2bb8a6c3fe07b1ea91e3c44265 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 20 Nov 2023 15:29:45 +0100 Subject: [PATCH 072/111] [delegtation_verify] Update the README. --- src/app/delegation_verify/README.md | 141 +++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 25 deletions(-) diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index 806ebcd85504..4f15c5d71cae 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -1,12 +1,39 @@ -This PR implements the stateless verification tool that would be used to verify that data collected by the uptime service. +Stateless delegation verification +--------------------------------- + +This is the stateless verification tool that would be used to verify +that data collected by the uptime service. It verifies the snark work +proofs that can optionally be attached to each submission and also it +validates the blocks claimed to be heads of submitters' best chains at +the time of each submission. + +The tool can work in two distinct modes: +* **file system mode**, where all submission and blocks should be + stored in files on a local file system; +* **cassandra mode**, where all the data is stored in a Cassandra + database. + +Note that submission verification and block validation which this tool +performs closely depends on the version of Mina it was compiled with. +Therefore it is important to always use the verifier tool compiled from +the same revision of the code which produced block and submission one +wants to verify. + +File system mode +---------------- Usage: ``` -./delegation-verify --block-dir ... -./delegation-verify --block-dir - +./delegation-verify fs --block-dir ... +./delegation-verify fs --block-dir - ``` -Where `` is a file path of the form `//-.json` -containing JSON following the format described in [Cloud storage section of delegation backend spec](https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage) (`` is a place holder for some directory): + +Where `` is a file path of the form +`//-.json` +containing JSON following the format described in +[Cloud storage section of delegation backend spec] +(https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage) +(`` is a place holder for some directory): ```json { "created_at": "server's timestamp (of the time of submission)" @@ -16,43 +43,107 @@ containing JSON following the format described in [Cloud storage section of dele , "block_hash": "base58check-encoded hash of a block" , "submitter": "base58check-encoded submitter's public key" } -``` +``` File path content: - `submitted_at_date` with server's date (of the time of submission) in format `YYYY-MM-DD` - `submitted_at` with server's timestamp (of the time of submission) in RFC-3339 - `submitter` is base58check-encoded submitter's public key -Parameter `--block-dir` specifies the path to the directory containing block for the submission. It is expected that blocks with `` exist under path `/.dat` for all of the filepaths provided. - -When `-` symbol is used as the `filename1`, no other `filename{i}` are accepted and all filenames will be read from the `stdin`, one filename per line. - -Utility `./delegation-verify` exits with the following exit codes: +Parameter `--block-dir` specifies the path to the directory containing +block for the submission. It is expected that blocks with +`` exist under path `/.dat` for all +of the filepaths provided. -- `0` if it was able to execute verification of all filepaths - - Note that `0` is returned even when some of the payloads were marked as invalid -- `1` if one of `` files violated the naming convention -- `2` if the utility failed to read some of `` files -- `3` if the utility failed to read some of `/.dat` files -- `5` if the other error occurred +When `-` symbol is used as the `filename1`, no other `filename{i}` are +accepted and all filenames will be read from the `stdin`, one filename +per line. -In case of non-zero exit code, some details of the error may be printed to `stderr`. - -For each `` parameter one line of output is printed to `stdout` : +For each `` parameter one line of output is printed to +`stdout` : - Valid payload at `` : - + ```json - { "state_hash": "" + { "created_at": "" + , "submitter": "" + , "parent": "" + , "state_hash": "" , "height": "" , "slot": "" } ``` - + - Invalid payload at `` : - + ```json { "error": "" } ``` -The stateless verification would check the protocol_state_proof in the block and the transaction_snark_proof in the snark_work. \ No newline at end of file + +The stateless verification would check the protocol_state_proof in the +block and the transaction_snark_proof in the snark_work. + +Cassandra mode +-------------- + +This mode of operations depends on an external program, `cqlsh` being +installed. It is a Cassandra client written in Python, which +delegation verifier uses to access Cassandra database. This program +should be available in the system path. If it's not, a path to the +`cqlsh` executable can be passed either in `CQLSH` environment +variable or through a command-line option `--clqsh`. Without access +to `cqlsh`, the delegation verifier can only work in the file system +mode described above. + +Additionally, the aforementioned Cassandra client requires some +configuration. In particular, and address and credentials to access +the right Cassandra server must be provided. They should be put in a +configuration file: `$HOME/.cassandra/cqlshrc`. The file can look +like this: + +``` +[authentication] +username = bpu-integration-test-at-673156464838 +password = ************************************ + +[connection] +hostname = cassandra.us-west-2.amazonaws.com +port = 9142 +ssl = true + +[ssl] +certfile = /home/user/uptime-service-backend/database/cert/sf-class2-root.crt +``` + +Usage: +``` +./delegation-verify cassandra --keyspace +``` + +Where: +* `keyspace` is the name of the Cassandra keyspace in the AWS cloud +* `start-timestamp` and `end-timestamp` define a time interval from which + submissions will be taken into account. They should be passed in the + format of the `submitted_at` field above (and in the database), that is: + `YYYY-MM-DD HH:mm:ss.0+0000`, where the trailing zeros describe the + timezone, which should be UTC. + +Given this command, the verifier will download submissions from the +given period one by one and verify them. Blocks will also be +downloaded from Cassandra and validated. For each submission, a JSON +statement will be output as shown in the previous +section. Additionally, also appropriate submission records in +Cassandra will be updated with the data. Any errors will also be +logged in Cassandra and **will not** cause the verifier to exit +with a non-zero exit code. + +Global options +-------------- + +Some command-line options work with both modes of operation: + +* `--no-check` – given this option, the program will skip validation + of blocks and snark work, and will immediately proceed to outputting + state hashes and other metadata for each submission, whether it is + valid or not. From 4f02ac9fbe3b6116a72a1fe7093137bb3912b11d Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 20 Nov 2023 16:18:56 +0100 Subject: [PATCH 073/111] [delegation_verify] Use core's dates_between function again. --- src/app/delegation_verify/submission.ml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index fc89ab8a60ae..67b4162053cc 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -186,19 +186,12 @@ module Cassandra = struct let submitter ({ submitter; _ } : submission) = submitter - let dates_between start finish = - let rec go acc current = - if Date.(current > finish) then acc - else go (current :: acc) Date.(add_days current 1) - in - go [] start - let load_submissions { executable; keyspace; period_start; period_end } = let open Deferred.Or_error.Let_syntax in let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in let partition_keys = - dates_between start_day end_day + Date.dates_between ~min:start_day ~max:end_day |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") in let partition = From d53da2e8164bb3528618b183e228b5f2f64fe352 Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 21 Nov 2023 13:15:23 +0000 Subject: [PATCH 074/111] PM-633 Remove debugging tools. add env vars and cassandra --- nix/docker.nix | 7 +++++-- nix/ocaml.nix | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nix/docker.nix b/nix/docker.nix index 61a67b194402..41d8c9e58a43 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,7 +1,8 @@ { lib, dockerTools, buildEnv, ocamlPackages_mina, runCommand, dumb-init , coreutils, bashInteractive, python3, libp2p_helper, procps, postgresql, curl -, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit }: +, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit, tzdata }: let + created = flockenzeit.lib.ISO-8601 currentTime; mkdir = name: @@ -81,10 +82,12 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; - contents = [ ocamlPackages_mina.mina-delegation-verify.out coreutils bashInteractive ]; + contents = [ ocamlPackages_mina.mina-delegation-verify.out ]; config = { cmd = [ "/bin/delegation-verify" ]; + Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" ]; }; + }; mina-image-full = mkFullImage "mina" (with ocamlPackages_mina; [ diff --git a/nix/ocaml.nix b/nix/ocaml.nix index defb546007a9..8a17f4fc9abd 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -98,8 +98,7 @@ let --set MINA_LIBP2P_HELPER_PATH ${pkgs.libp2p_helper}/bin/libp2p_helper \ --set MINA_COMMIT_SHA1 ${escapeShellArg commit_sha1} \ --set MINA_COMMIT_DATE ${escapeShellArg commit_date} \ - --set MINA_BRANCH "''${MINA_BRANCH-}" \ - --set TZDIR "${pkgs.tzdata}/share/zoneinfo" + --set MINA_BRANCH "''${MINA_BRANCH-}" done '') package.outputs); @@ -341,7 +340,7 @@ let ''; }); - mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; + mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { deps = [ pkgs.cassandra_4 ]; }; # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { From 0501c57e4000e300892b9600ad194debbfe39d88 Mon Sep 17 00:00:00 2001 From: Smorci Date: Wed, 22 Nov 2023 14:26:49 +0000 Subject: [PATCH 075/111] PM-633 Add outputs to README --- nix/README.md | 27 +++++++++++++++++++++++---- nix/docker.nix | 1 - src/app/delegation_verify/README.md | 5 +++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/nix/README.md b/nix/README.md index 725f98976ac2..09bbc655be08 100644 --- a/nix/README.md +++ b/nix/README.md @@ -182,6 +182,23 @@ Alternatively, you can build the entirety of Mina inside the Nix sandbox fully automatically: run `nix build mina` if you're using flakes (or `nix-build` otherwise). You can find the resulting executable at `result/bin/mina`. +To build the available packages use `nix build mina#` where `` is from the following list. + +Available packages: +- **mina** - mina daemon executable +- **mina-deb** - mina debian package +- **mina_tests** - mina unit tests +- **mina-ocaml-format** - checks if the code is formatted properly +- **mina-delegation-verify** - [stateless verification tool](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/delegation_verify) for Delegation Program +- **test_executive** - [test executive](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/test_executive) executable +- **libp2p_helper** - helper go module for unit tests +- **kimchi_bindings_stubs** +- **snarky_js** - builds snarkyjs npm package +- **leaderboard** - builds the [leaderboard](https://github.com/MinaProtocol/mina/tree/berkeley/frontend/leaderboard) yarn package +- **validation** - builds the [validation](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/validation) app +- **trace-tool** - builds the [trace-tool](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/trace-tool) +- **zkapp-cli** - builds the [zkapp-cli](https://github.com/o1-labs/zkapp-cli/) + ### "Impure" development shell TL;DR: @@ -210,7 +227,6 @@ branches, or otherwise changing the dependency tree of Mina. TL;DR: ``` $(nix build mina#mina-image-full) | docker load -# Also available: mina-image-slim, mina-archive-image-full ``` Since a "pure" build can happen entirely inside the Nix sandbox, we can use its @@ -225,8 +241,11 @@ registry at `docker run --rm -it us-west2-docker.pkg.dev/o1labs-192920/nix-containers/mina-image-full:develop` . -The `slim` image only has the Mina daemon itself, whereas `full` images also -contain many useful tools, such as coreutils, fake init, jq, etc. +Available images: +- **mina-image-full** - full Mina daemon image with useful tools, such as coreutils, fake init, jq, etc. +- **mina-image-slim** - has Mina daemon only +- **mina-archive-image-full** - full Mina archive image with useful tools +- **mina-delegation-verify-image** - stateless verification tool for Delegation Program ### Debian package @@ -601,4 +620,4 @@ Before running any `dune` commands. Alternatively, you can just run your commands inside `nix develop --ignore-environment mina`, which unsets all the outside environment variables, -resulting in a more reproducible but less convenient environment. \ No newline at end of file +resulting in a more reproducible but less convenient environment. diff --git a/nix/docker.nix b/nix/docker.nix index 41d8c9e58a43..6a3853c0f755 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -87,7 +87,6 @@ in { cmd = [ "/bin/delegation-verify" ]; Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" ]; }; - }; mina-image-full = mkFullImage "mina" (with ocamlPackages_mina; [ diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index 4f15c5d71cae..506a022bd93d 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -147,3 +147,8 @@ Some command-line options work with both modes of operation: of blocks and snark work, and will immediately proceed to outputting state hashes and other metadata for each submission, whether it is valid or not. + +Build +----- + +The executable and docker image can be built with Nix. For further instructions, consult the Mina [Nix documentation](https://github.com/MinaProtocol/mina/tree/berkeley/nix) From 16f01d2cd6f928b3e4443ecac89de75370c4d766 Mon Sep 17 00:00:00 2001 From: Smorci Date: Fri, 24 Nov 2023 14:02:53 +0000 Subject: [PATCH 076/111] Remove stateless verifier dockerfile --- .../Dockerfile-delegation-stateless-verifier | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 dockerfiles/Dockerfile-delegation-stateless-verifier diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier deleted file mode 100644 index e607dc6b0efe..000000000000 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ /dev/null @@ -1,45 +0,0 @@ -################################################################################################# -# The "stateless verification build" Stage -# - builds stateless verification tool -# - should not include any data related to joining a specific network, only the node software itself -################################################################################################# -FROM gcr.io/o1labs-192920/mina-toolchain@sha256:73562fcc35dcabd342f66f1d69ae12704e92d69edc0b37e7c88b4d11bc623f23 AS builder - -# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI -ARG DUNE_PROFILE=devnet - -# branch to checkout on first clone (this will be the only availible branch in the container) -# can also be a tagged release -ARG MINA_BRANCH=compatible - -# repo to checkout the branch from -ARG MINA_REPO=https://github.com/MinaProtocol/mina - -# location of repo used for pins and external package commits -ARG MINA_DIR=mina - -ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" - -# git will clone into an empty dir, but this also helps us set the workdir in advance -RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ - && git clone \ - -b "${MINA_BRANCH}" \ - --depth 1 \ - --shallow-submodules \ - --recurse-submodules \ - ${MINA_REPO} ${HOME}/${MINA_DIR} - -WORKDIR $HOME/${MINA_DIR} - -RUN mkdir ${HOME}/app - -# HACK: build without special cpu features to allow more people to run delegation verification tool -RUN ./scripts/zexe-standardize.sh - -RUN eval $(opam config env) \ - && dune build --profile=${DUNE_PROFILE} \ - src/app/delegation_verify/delegation_verify.exe \ - && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ - && rm -rf _build - -ENTRYPOINT ["./delegation-verify"] From 7c6ac0116065fc094550a9d9818fef7d5c91518e Mon Sep 17 00:00:00 2001 From: Smorci Date: Fri, 24 Nov 2023 14:04:20 +0000 Subject: [PATCH 077/111] Add coreutils and bash to delegation verify docker image --- nix/README.md | 2 +- nix/docker.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/README.md b/nix/README.md index 09bbc655be08..44e143ca245c 100644 --- a/nix/README.md +++ b/nix/README.md @@ -226,7 +226,7 @@ branches, or otherwise changing the dependency tree of Mina. TL;DR: ``` -$(nix build mina#mina-image-full) | docker load +$(nix build mina#mina-image-full --print-out-paths ) | docker load ``` Since a "pure" build can happen entirely inside the Nix sandbox, we can use its diff --git a/nix/docker.nix b/nix/docker.nix index 6a3853c0f755..cc501ccba2f0 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -82,7 +82,7 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; - contents = [ ocamlPackages_mina.mina-delegation-verify.out ]; + contents = [ ocamlPackages_mina.mina-delegation-verify.out coreutils bashInteractive ]; config = { cmd = [ "/bin/delegation-verify" ]; Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" ]; From a64c1f4553d0565d3db1e53521ca21bc9d8178e8 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Mon, 27 Nov 2023 12:42:46 +0100 Subject: [PATCH 078/111] [delegation_verify] Allow configuration through env variables. --- src/app/delegation_verify/README.md | 11 +++- src/app/delegation_verify/cassandra.ml | 54 +++++++++++++++---- src/app/delegation_verify/cassandra.mli | 10 ++-- .../delegation_verify/delegation_verify.ml | 3 +- src/app/delegation_verify/submission.ml | 17 +++--- 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index 506a022bd93d..fdd334fcddf4 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -97,7 +97,7 @@ to `cqlsh`, the delegation verifier can only work in the file system mode described above. Additionally, the aforementioned Cassandra client requires some -configuration. In particular, and address and credentials to access +configuration. In particular, an address and credentials to access the right Cassandra server must be provided. They should be put in a configuration file: `$HOME/.cassandra/cqlshrc`. The file can look like this: @@ -116,6 +116,15 @@ ssl = true certfile = /home/user/uptime-service-backend/database/cert/sf-class2-root.crt ``` +Alternatively this configuration can be provided in environment variables: +* `CASSANDRA_HOST` +* `CASSANDRA_PORT` +* `CASSANDRA_USERNAME` +* `CASSANDRA_PASSWORD` +* `SSL_CERTFILE` +* `CASSANDRA_USE_SSL` – "1", "YES", "yes", "TRUE", "true" all mean yes, + any other value means no. + Usage: ``` ./delegation-verify cassandra --keyspace diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 8b906e6bf274..c2cc88350ea8 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -3,20 +3,56 @@ open Core type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -let query ?executable q = - let prog = +type connection_conf = + { hostname : string + ; port : int + ; username : string + ; password : string + ; ssl : bool + } + +type conf = + { executable : string + ; connection : connection_conf option + ; keyspace : string + } + +let make_conn_conf () : connection_conf option = + let open Option.Let_syntax in + let%bind hostname = Sys.getenv "CASSANDRA_HOST" in + let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in + let%bind ssl = + Sys.getenv "CASSANDRA_USE_SSL" + |> Option.map ~f:(List.mem ~equal:String.equal ["1"; "TRUE"; "true"; "YES"; "yes"]) + in + let%bind username = Sys.getenv "CASSANDRA_USERNAME" in + let%map password = Sys.getenv "CASSANDRA_PASSWORD" in + { hostname; port; ssl; username; password } + +let make_conf ?executable ~keyspace : conf = + let conn = make_conn_conf () in + let executable = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in - Process.run_lines ~prog ~stdin:q ~args:[] () + { executable; connection = conn; keyspace } + +let query ~conf q = + let args = + Option.value_map conf.connection ~default:[] + ~f:(fun { hostname; port; ssl; username; password } -> + [ "--username"; username; "--password"; password + ; hostname; Int.to_string port ] @ ( if ssl then [ "--ssl" ] else [] )) + in + Process.run_lines ~prog:conf.executable ~stdin:q ~args () -let select ?executable ~keyspace ~parse ~fields ?where from = +let select ~conf ~parse ~fields ?where from = let open Deferred.Or_error.Let_syntax in let%bind data = - query ?executable + query ~conf @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" (String.concat ~sep:"," fields) - keyspace from + conf.keyspace from (match where with None -> "" | Some w -> " WHERE " ^ w) in List.slice data 3 (-2) (* skip header and footer *) @@ -34,13 +70,13 @@ let select ?executable ~keyspace ~parse ~fields ?where from = with Yojson.Json_error e -> Or_error.error_string e ) |> Deferred.return -let update ?executable ~keyspace ~table ~where updates = +let update ~conf ~table ~where updates = let open Deferred.Or_error.Let_syntax in let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in let%map _ = - query ?executable + query ~conf @@ Printf.sprintf "CONSISTENCY LOCAL_QUORUM; UPDATE %s.%s SET %s WHERE %s;" - keyspace table + conf.keyspace table (String.concat ~sep:"," assignments) where in diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli index 990bd0de6b90..d2f6247352f1 100644 --- a/src/app/delegation_verify/cassandra.mli +++ b/src/app/delegation_verify/cassandra.mli @@ -2,9 +2,12 @@ open Async type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or +type conf + +val make_conf : ?executable:string -> keyspace:string -> conf + val select : - ?executable:string - -> keyspace:string + conf:conf -> parse:'a parser -> fields:string list -> ?where:string @@ -12,8 +15,7 @@ val select : -> 'a list Deferred.Or_error.t val update : - ?executable:string - -> keyspace:string + conf:conf -> table:string -> where:string -> (string * string) list diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index d478bf8a385d..462558f13df4 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -199,8 +199,7 @@ let cassandra_command = end) in let src = Submission.Cassandra. - { executable = cqlsh - ; keyspace + { conf = Cassandra.make_conf ?executable:cqlsh ~keyspace ; period_start ; period_end } diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 67b4162053cc..46e5e335aed7 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -126,8 +126,7 @@ end module Cassandra = struct type t = - { executable : string option - ; keyspace : string + { conf : Cassandra.conf ; period_start : string ; period_end : string } @@ -186,7 +185,7 @@ module Cassandra = struct let submitter ({ submitter; _ } : submission) = submitter - let load_submissions { executable; keyspace; period_start; period_end } = + let load_submissions { conf ; period_start; period_end } = let open Deferred.Or_error.Let_syntax in let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in @@ -203,7 +202,7 @@ module Cassandra = struct (String.concat ~sep:"," @@ List.map ~f:(sprintf "'%s'") partition_keys) in let%bind raw = - Cassandra.select ?executable ~parse:raw_of_yojson ~keyspace + Cassandra.select ~conf ~parse:raw_of_yojson ~fields: [ "created_at" ; "submitted_at_date" @@ -234,10 +233,10 @@ module Cassandra = struct s :: l ) |> Deferred.return - let load_block ~block_hash { executable; keyspace; _ } = + let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in let%bind block_data = - Cassandra.select ?executable ~parse:block_data_of_yojson ~keyspace + Cassandra.select ~conf ~parse:block_data_of_yojson ~fields:[ "raw_block" ] ~where:(sprintf "block_hash = '%s'" block_hash) "blocks" @@ -249,10 +248,10 @@ module Cassandra = struct String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return - let output src (submission : submission) = function + let output { conf; _ } (submission : submission) = function | Ok payload -> Output.display payload ; - Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + Cassandra.update ~conf ~table:"submissions" ~where: (sprintf @@ -264,7 +263,7 @@ module Cassandra = struct Output.(valid_payload_to_cassandra_updates payload) | Error e -> Output.display_error @@ Error.to_string_hum e ; - Cassandra.update ?executable:src.executable ~keyspace:src.keyspace + Cassandra.update ~conf ~table:"submissions" ~where: (sprintf From 52ed56774c1f5e68293efec4d8b701ee3b82cb7f Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 29 Nov 2023 14:29:54 +0100 Subject: [PATCH 079/111] [delegation_verify] Allow for partial configuration with env vars. --- src/app/delegation_verify/README.md | 5 +++++ src/app/delegation_verify/cassandra.ml | 30 +++++++++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index fdd334fcddf4..97a316c525e3 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -156,6 +156,11 @@ Some command-line options work with both modes of operation: of blocks and snark work, and will immediately proceed to outputting state hashes and other metadata for each submission, whether it is valid or not. + +* `--config-file` - a configuration file of the Mina network. It is + necessary to provide it if the network which produced the blocks + used configuration affecting block validation. In most cases it can + be omitted. Build ----- diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index c2cc88350ea8..b56c9609a108 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -6,14 +6,18 @@ type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or type connection_conf = { hostname : string ; port : int - ; username : string - ; password : string ; ssl : bool } +type credentials = + { username : string + ; password : string + } + type conf = { executable : string ; connection : connection_conf option + ; credentials : credentials option ; keyspace : string } @@ -21,28 +25,34 @@ let make_conn_conf () : connection_conf option = let open Option.Let_syntax in let%bind hostname = Sys.getenv "CASSANDRA_HOST" in let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in - let%bind ssl = + let%map ssl = Sys.getenv "CASSANDRA_USE_SSL" |> Option.map ~f:(List.mem ~equal:String.equal ["1"; "TRUE"; "true"; "YES"; "yes"]) in + { hostname; port; ssl } + +let make_cred_conf () : credentials option = + let open Option.Let_syntax in let%bind username = Sys.getenv "CASSANDRA_USERNAME" in let%map password = Sys.getenv "CASSANDRA_PASSWORD" in - { hostname; port; ssl; username; password } - + { username; password } + let make_conf ?executable ~keyspace : conf = let conn = make_conn_conf () in + let credentials = make_cred_conf () in let executable = Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const |> Option.value ~default:"cqlsh" in - { executable; connection = conn; keyspace } + { executable; connection = conn; credentials; keyspace } let query ~conf q = + let optional ~f = Option.value_map ~f ~default:[] in let args = - Option.value_map conf.connection ~default:[] - ~f:(fun { hostname; port; ssl; username; password } -> - [ "--username"; username; "--password"; password - ; hostname; Int.to_string port ] @ ( if ssl then [ "--ssl" ] else [] )) + optional conf.credentials ~f:(fun { username; password } -> + [ "--username"; username; "--password"; password ]) + @ optional conf.connection ~f:(fun { hostname; port; ssl } -> + ( if ssl then [ "--ssl" ] else [] ) @ [ hostname; Int.to_string port ]) in Process.run_lines ~prog:conf.executable ~stdin:q ~args () From f83f715d6c96b7ddb5c86171c55a1648fb229e44 Mon Sep 17 00:00:00 2001 From: Smorci Date: Thu, 30 Nov 2023 14:45:27 +0000 Subject: [PATCH 080/111] Add cqlsh expansion python packages to delegation verifier binary --- flake.nix | 1 + nix/docker.nix | 64 +++++++++++++++++++++++++++----------------------- nix/ocaml.nix | 8 +++---- nix/python.nix | 38 ++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 34 deletions(-) create mode 100644 nix/python.nix diff --git a/flake.nix b/flake.nix index 0136412c2158..725ac2846c90 100644 --- a/flake.nix +++ b/flake.nix @@ -69,6 +69,7 @@ rust = import ./nix/rust.nix; go = import ./nix/go.nix; javascript = import ./nix/javascript.nix; + python = import ./nix/python.nix; ocaml = final: prev: { ocamlPackages_mina = requireSubmodules (import ./nix/ocaml.nix { inherit inputs; diff --git a/nix/docker.nix b/nix/docker.nix index cc501ccba2f0..e0e5e6bfe08e 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -48,29 +48,30 @@ let ''; }; - mkFullImage = name: packages: dockerTools.streamLayeredImage { - name = "${name}-full"; - inherit created; - contents = [ - dumb-init - coreutils - bashInteractive - python3 - libp2p_helper - procps - curl - jq - ] ++ packages; - extraCommands = '' - mkdir root tmp - chmod 777 tmp - ''; - config = { - env = [ "MINA_TIME_OFFSET=0" ]; - WorkingDir = "/root"; - cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; + mkFullImage = name: packages: + dockerTools.streamLayeredImage { + name = "${name}-full"; + inherit created; + contents = [ + dumb-init + coreutils + bashInteractive + python3 + libp2p_helper + procps + curl + jq + ] ++ packages; + extraCommands = '' + mkdir root tmp + chmod 777 tmp + ''; + config = { + env = [ "MINA_TIME_OFFSET=0" ]; + WorkingDir = "/root"; + cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; + }; }; - }; in { mina-image-slim = dockerTools.streamLayeredImage { @@ -82,7 +83,11 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; - contents = [ ocamlPackages_mina.mina-delegation-verify.out coreutils bashInteractive ]; + contents = [ + ocamlPackages_mina.mina-delegation-verify.out + coreutils + bashInteractive + ]; config = { cmd = [ "/bin/delegation-verify" ]; Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" ]; @@ -97,11 +102,12 @@ in { mina.mainnet mina.genesis ]); - mina-archive-image-full = mkFullImage "mina-archive" (with ocamlPackages_mina; [ - mina-archive-scripts - gnutar - gzip + mina-archive-image-full = mkFullImage "mina-archive" + (with ocamlPackages_mina; [ + mina-archive-scripts + gnutar + gzip - mina.archive - ]); + mina.archive + ]); } diff --git a/nix/ocaml.nix b/nix/ocaml.nix index 8a17f4fc9abd..0af21558283b 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -322,15 +322,13 @@ let }); # Stateless verification tool - mina-delegation-verify-dev = self.mina-dev.overrideAttrs (oa: { + mina-delegation-verify-dev = self.mina-dev.overrideAttrs (_: { pname = "mina-delegation-verify"; version = "dev"; outputs = [ "out" ]; - buildInputs = oa.buildInputs ++ [ pkgs.cassandra_4 ]; - buildPhase = '' - dune build --display=short \ + dune build --display=short --profile=devnet \ src/app/delegation_verify/delegation_verify.exe ''; @@ -340,7 +338,7 @@ let ''; }); - mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { deps = [ pkgs.cassandra_4 ]; }; + mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { diff --git a/nix/python.nix b/nix/python.nix new file mode 100644 index 000000000000..e8f66d6c27b8 --- /dev/null +++ b/nix/python.nix @@ -0,0 +1,38 @@ +# An overlay defining Python packages +final: prev: { + + cassandra-sigv4 = final.python3Packages.buildPythonPackage rec { + pname = "cassandra-sigv4"; + version = "4.0.2"; + pyproject = true; + src = final.python3Packages.fetchPypi { + inherit pname version; + hash = "sha256-d/9hI6uUHFCnCuUXed5ae74oQ9lrgMRmrP/fJil0ipg="; + }; + nativeBuildInputs = with prev.python3Packages; [ + setuptools-scm + six + cassandra-driver + boto3 + mock + ]; + doCheck = false; + }; + + cqlsh-expansion = final.python3Packages.buildPythonPackage rec { + pname = "cqlsh-expansion"; + version = "0.9.6"; + pyproject = true; + src = final.python3Packages.fetchPypi { + inherit pname version; + hash = "sha256-ea5ew7aF19JcdzoMgLrvlj0GC/wLfbIsABLKRxcT6DQ="; + }; + nativeBuildInputs = with final.python3Packages; [ + setuptools-scm + cassandra-driver + final.cassandra-sigv4 + boto3 + ]; + doCheck = false; + }; +} From 74a1b776fcf65ac3803336025277e166c21cd229 Mon Sep 17 00:00:00 2001 From: Smorci Date: Thu, 30 Nov 2023 14:45:27 +0000 Subject: [PATCH 081/111] Add cqlsh expansion python packages to delegation verifier binary --- flake.nix | 1 + nix/docker.nix | 70 ++++++++++++++++++++++++++++---------------------- nix/ocaml.nix | 8 +++--- nix/python.nix | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 36 deletions(-) create mode 100644 nix/python.nix diff --git a/flake.nix b/flake.nix index 0136412c2158..725ac2846c90 100644 --- a/flake.nix +++ b/flake.nix @@ -69,6 +69,7 @@ rust = import ./nix/rust.nix; go = import ./nix/go.nix; javascript = import ./nix/javascript.nix; + python = import ./nix/python.nix; ocaml = final: prev: { ocamlPackages_mina = requireSubmodules (import ./nix/ocaml.nix { inherit inputs; diff --git a/nix/docker.nix b/nix/docker.nix index cc501ccba2f0..7a0182f5ecbb 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,6 +1,7 @@ { lib, dockerTools, buildEnv, ocamlPackages_mina, runCommand, dumb-init , coreutils, bashInteractive, python3, libp2p_helper, procps, postgresql, curl -, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit, tzdata }: +, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit, tzdata +, cqlsh-expansion, python3Packages }: let created = flockenzeit.lib.ISO-8601 currentTime; @@ -48,29 +49,30 @@ let ''; }; - mkFullImage = name: packages: dockerTools.streamLayeredImage { - name = "${name}-full"; - inherit created; - contents = [ - dumb-init - coreutils - bashInteractive - python3 - libp2p_helper - procps - curl - jq - ] ++ packages; - extraCommands = '' - mkdir root tmp - chmod 777 tmp - ''; - config = { - env = [ "MINA_TIME_OFFSET=0" ]; - WorkingDir = "/root"; - cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; + mkFullImage = name: packages: + dockerTools.streamLayeredImage { + name = "${name}-full"; + inherit created; + contents = [ + dumb-init + coreutils + bashInteractive + python3 + libp2p_helper + procps + curl + jq + ] ++ packages; + extraCommands = '' + mkdir root tmp + chmod 777 tmp + ''; + config = { + env = [ "MINA_TIME_OFFSET=0" ]; + WorkingDir = "/root"; + cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; + }; }; - }; in { mina-image-slim = dockerTools.streamLayeredImage { @@ -82,10 +84,15 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; - contents = [ ocamlPackages_mina.mina-delegation-verify.out coreutils bashInteractive ]; + contents = [ + ocamlPackages_mina.mina-delegation-verify.out + cqlsh-expansion + coreutils + bashInteractive + ]; config = { cmd = [ "/bin/delegation-verify" ]; - Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" ]; + Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" "CQLSH=${cqlsh-expansion}/bin/cqlsh-expansion" "USER_SITE=${cqlsh-expansion}/${python3Packages.python.sitePackages}" ]; }; }; @@ -97,11 +104,12 @@ in { mina.mainnet mina.genesis ]); - mina-archive-image-full = mkFullImage "mina-archive" (with ocamlPackages_mina; [ - mina-archive-scripts - gnutar - gzip + mina-archive-image-full = mkFullImage "mina-archive" + (with ocamlPackages_mina; [ + mina-archive-scripts + gnutar + gzip - mina.archive - ]); + mina.archive + ]); } diff --git a/nix/ocaml.nix b/nix/ocaml.nix index 8a17f4fc9abd..0af21558283b 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -322,15 +322,13 @@ let }); # Stateless verification tool - mina-delegation-verify-dev = self.mina-dev.overrideAttrs (oa: { + mina-delegation-verify-dev = self.mina-dev.overrideAttrs (_: { pname = "mina-delegation-verify"; version = "dev"; outputs = [ "out" ]; - buildInputs = oa.buildInputs ++ [ pkgs.cassandra_4 ]; - buildPhase = '' - dune build --display=short \ + dune build --display=short --profile=devnet \ src/app/delegation_verify/delegation_verify.exe ''; @@ -340,7 +338,7 @@ let ''; }); - mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { deps = [ pkgs.cassandra_4 ]; }; + mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { diff --git a/nix/python.nix b/nix/python.nix new file mode 100644 index 000000000000..abd8b542181c --- /dev/null +++ b/nix/python.nix @@ -0,0 +1,46 @@ +# An overlay defining Python packages +final: prev: { + + cassandra-sigv4 = final.python3Packages.buildPythonPackage rec { + pname = "cassandra-sigv4"; + version = "4.0.2"; + pyproject = true; + src = final.python3Packages.fetchPypi { + inherit pname version; + hash = "sha256-d/9hI6uUHFCnCuUXed5ae74oQ9lrgMRmrP/fJil0ipg="; + }; + nativeBuildInputs = with prev.python3Packages; [ + setuptools-scm + six + cassandra-driver + boto3 + mock + ]; + doCheck = false; + }; + + cqlsh-expansion = final.python3Packages.buildPythonPackage rec { + pname = "cqlsh-expansion"; + version = "0.9.6"; + pyproject = true; + src = final.python3Packages.fetchPypi { + inherit pname version; + hash = "sha256-ea5ew7aF19JcdzoMgLrvlj0GC/wLfbIsABLKRxcT6DQ="; + }; + buildInputs = with final.python3Packages; [ + cassandra-driver + six + setuptools-scm + final.cassandra-sigv4 + boto3 + ]; + pythonPath = with final.python3Packages; [ + cassandra-driver + six + final.cassandra-sigv4 + boto3 + ]; + permitUserSite = true; + doCheck = false; + }; +} From 5d2ec4e5a8a1be5e2ab9f3a7ad07951113511549 Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 5 Dec 2023 13:47:28 +0100 Subject: [PATCH 082/111] [delegation_verify] Use different Cassandra field to identify records. --- src/app/delegation_verify/cassandra.ml | 26 ++++----- .../delegation_verify/delegation_verify.ml | 2 +- src/app/delegation_verify/output.ml | 4 +- src/app/delegation_verify/submission.ml | 54 +++++++++---------- 4 files changed, 37 insertions(+), 49 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index b56c9609a108..d32b112ef477 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -3,16 +3,9 @@ open Core type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -type connection_conf = - { hostname : string - ; port : int - ; ssl : bool - } +type connection_conf = { hostname : string; port : int; ssl : bool } -type credentials = - { username : string - ; password : string - } +type credentials = { username : string; password : string } type conf = { executable : string @@ -25,9 +18,10 @@ let make_conn_conf () : connection_conf option = let open Option.Let_syntax in let%bind hostname = Sys.getenv "CASSANDRA_HOST" in let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in - let%map ssl = + let%map ssl = Sys.getenv "CASSANDRA_USE_SSL" - |> Option.map ~f:(List.mem ~equal:String.equal ["1"; "TRUE"; "true"; "YES"; "yes"]) + |> Option.map + ~f:(List.mem ~equal:String.equal [ "1"; "TRUE"; "true"; "YES"; "yes" ]) in { hostname; port; ssl } @@ -36,7 +30,7 @@ let make_cred_conf () : credentials option = let%bind username = Sys.getenv "CASSANDRA_USERNAME" in let%map password = Sys.getenv "CASSANDRA_PASSWORD" in { username; password } - + let make_conf ?executable ~keyspace : conf = let conn = make_conn_conf () in let credentials = make_cred_conf () in @@ -49,10 +43,10 @@ let make_conf ?executable ~keyspace : conf = let query ~conf q = let optional ~f = Option.value_map ~f ~default:[] in let args = - optional conf.credentials ~f:(fun { username; password } -> - [ "--username"; username; "--password"; password ]) - @ optional conf.connection ~f:(fun { hostname; port; ssl } -> - ( if ssl then [ "--ssl" ] else [] ) @ [ hostname; Int.to_string port ]) + optional conf.credentials ~f:(fun { username; password } -> + [ "--username"; username; "--password"; password ] ) + @ optional conf.connection ~f:(fun { hostname; port; ssl } -> + (if ssl then [ "--ssl" ] else []) @ [ hostname; Int.to_string port ] ) in Process.run_lines ~prog:conf.executable ~stdin:q ~args () diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 462558f13df4..c0a036ce16a8 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -126,7 +126,7 @@ module Make_verifier (Source : Submission.Data_source) = struct let%bind result = verify ~validate submission in Result.map result ~f:(fun (state_hash, parent, height, slot) -> Output. - { created_at = Source.created_at submission + { submitted_at = Source.submitted_at submission ; submitter = Signature_lib.Public_key.Compressed.to_base58_check (Source.submitter submission) diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml index 1d98d1dd55f4..e350715a9871 100644 --- a/src/app/delegation_verify/output.ml +++ b/src/app/delegation_verify/output.ml @@ -2,7 +2,7 @@ open Async open Mina_base type t = - { created_at : string + { submitted_at : string ; submitter : string ; state_hash : State_hash.t ; parent : State_hash.t @@ -12,7 +12,7 @@ type t = let valid_payload_to_yojson (p : t) : Yojson.Safe.t = `Assoc - [ ("created_at", `String p.created_at) + [ ("submitted_at", `String p.submitted_at) ; ("submitter", `String p.submitter) ; ("state_hash", State_hash.to_yojson p.state_hash) ; ("parent", State_hash.to_yojson p.parent) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 46e5e335aed7..5fb94232e820 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -15,7 +15,7 @@ module type Data_source = sig type submission - val created_at : submission -> string + val submitted_at : submission -> string val block_hash : submission -> string @@ -44,14 +44,14 @@ module Filesystem = struct type t = { block_dir : string; submission_paths : string list } type submission = - { created_at : string + { submitted_at : string ; snark_work : Uptime_service.Proof_data.t option ; submitter : Public_key.Compressed.t ; block_hash : string } type raw = - { created_at : string + { submitted_at : string ; peer_id : string ; snark_work : string option [@default None] ; remote_addr : string @@ -77,10 +77,10 @@ module Filesystem = struct { submitter ; snark_work ; block_hash = meta.block_hash - ; created_at = meta.created_at + ; submitted_at = meta.submitted_at } - let created_at ({ created_at; _ } : submission) = created_at + let submitted_at ({ submitted_at; _ } : submission) = submitted_at let block_hash ({ block_hash; _ } : submission) = block_hash @@ -125,11 +125,7 @@ module Filesystem = struct end module Cassandra = struct - type t = - { conf : Cassandra.conf - ; period_start : string - ; period_end : string - } + type t = { conf : Cassandra.conf; period_start : string; period_end : string } type block_data = { raw_block : string } [@@deriving yojson] @@ -177,7 +173,7 @@ module Cassandra = struct } : submission ) - let created_at ({ created_at; _ } : submission) = created_at + let submitted_at ({ submitted_at; _ } : submission) = submitted_at let block_hash ({ block_hash; _ } : submission) = block_hash @@ -185,10 +181,14 @@ module Cassandra = struct let submitter ({ submitter; _ } : submission) = submitter - let load_submissions { conf ; period_start; period_end } = + let load_submissions { conf; period_start; period_end } = let open Deferred.Or_error.Let_syntax in - let start_day = Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc in - let end_day = Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc in + let start_day = + Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc + in + let end_day = + Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc + in let partition_keys = Date.dates_between ~min:start_day ~max:end_day |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") @@ -197,8 +197,7 @@ module Cassandra = struct if List.length partition_keys = 1 then sprintf "submitted_at_date = '%s'" (List.hd_exn partition_keys) else - sprintf - "submitted_at_date IN (%s)" + sprintf "submitted_at_date IN (%s)" (String.concat ~sep:"," @@ List.map ~f:(sprintf "'%s'") partition_keys) in let%bind raw = @@ -215,10 +214,8 @@ module Cassandra = struct ; "graphql_control_port" ] ~where: - (sprintf - "%s AND submitted_at >= '%s' AND submitted_at <= '%s'" - partition - period_start period_end ) + (sprintf "%s AND submitted_at >= '%s' AND submitted_at <= '%s'" + partition period_start period_end ) "submissions" in List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> @@ -236,8 +233,7 @@ module Cassandra = struct let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in let%bind block_data = - Cassandra.select ~conf ~parse:block_data_of_yojson - ~fields:[ "raw_block" ] + Cassandra.select ~conf ~parse:block_data_of_yojson ~fields:[ "raw_block" ] ~where:(sprintf "block_hash = '%s'" block_hash) "blocks" in @@ -251,26 +247,24 @@ module Cassandra = struct let output { conf; _ } (submission : submission) = function | Ok payload -> Output.display payload ; - Cassandra.update ~conf - ~table:"submissions" + Cassandra.update ~conf ~table:"submissions" ~where: (sprintf "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ = '%s'" - (List.hd_exn @@ String.split ~on:' ' submission.created_at) - submission.created_at + (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) + submission.submitted_at (Public_key.Compressed.to_base58_check submission.submitter) ) Output.(valid_payload_to_cassandra_updates payload) | Error e -> Output.display_error @@ Error.to_string_hum e ; - Cassandra.update ~conf - ~table:"submissions" + Cassandra.update ~conf ~table:"submissions" ~where: (sprintf "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ = '%s'" - (List.hd_exn @@ String.split ~on:' ' submission.created_at) - submission.created_at + (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) + submission.submitted_at (Public_key.Compressed.to_base58_check submission.submitter) ) [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) ] end From 8543f09ea8524a54aada9e0b8da7fa37310c5e5b Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 5 Dec 2023 15:22:31 +0100 Subject: [PATCH 083/111] [delegation_verify] Make the upper interval bound exclusive. Co-authored-by: piotr-iohk <42900201+piotr-iohk@users.noreply.github.com> --- src/app/delegation_verify/submission.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 5fb94232e820..95771dd4398e 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -214,7 +214,7 @@ module Cassandra = struct ; "graphql_control_port" ] ~where: - (sprintf "%s AND submitted_at >= '%s' AND submitted_at <= '%s'" + (sprintf "%s AND submitted_at >= '%s' AND submitted_at < '%s'" partition period_start period_end ) "submissions" in From 1c16a940bea44d3bb6828cab3928049fad25728d Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 5 Dec 2023 14:39:25 +0000 Subject: [PATCH 084/111] Add mina delegation verify init derivation --- nix/docker.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nix/docker.nix b/nix/docker.nix index 39bdb849b19a..972ca449545d 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -49,6 +49,17 @@ let ''; }; + mina-delegation-verify-init = runCommand "mina-delegation-verify-init" {} '' + + mkdir -p $out + + export HOME=$out + export PYTHONUSERBASE=${cqlsh-expansion} + + ${cqlsh-expansion}/bin/cqlsh-expansion.init + + ''; + mkFullImage = name: packages: dockerTools.streamLayeredImage { name = "${name}-full"; @@ -87,11 +98,14 @@ in { contents = [ ocamlPackages_mina.mina-delegation-verify.out cqlsh-expansion + mina-delegation-verify-init coreutils bashInteractive ]; config = { - cmd = [ "/bin/delegation-verify" ]; + cmd = [ + "bash" + ]; Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" "CQLSH=${cqlsh-expansion}/bin/cqlsh-expansion" "PYTHONUSERBASE=${cqlsh-expansion}" ]; }; }; From f137fb3e960b4b2fa4b53b2936c3832ddc97834b Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 5 Dec 2023 14:51:51 +0000 Subject: [PATCH 085/111] Cleanup and formatting nix code --- nix/docker.nix | 18 ++++++++++-------- nix/ocaml.nix | 3 ++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nix/docker.nix b/nix/docker.nix index 972ca449545d..ec01c4fda943 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -49,14 +49,14 @@ let ''; }; - mina-delegation-verify-init = runCommand "mina-delegation-verify-init" {} '' + mina-delegation-verify-init = runCommand "mina-delegation-verify-init" { } '' - mkdir -p $out + mkdir -p $out - export HOME=$out - export PYTHONUSERBASE=${cqlsh-expansion} + export HOME=$out + export PYTHONUSERBASE=${cqlsh-expansion} - ${cqlsh-expansion}/bin/cqlsh-expansion.init + ${cqlsh-expansion}/bin/cqlsh-expansion.init ''; @@ -103,10 +103,12 @@ in { bashInteractive ]; config = { - cmd = [ - "bash" + cmd = [ "bash" ]; + Env = [ + "TZ=Etc/UTC" + "TZDIR=${tzdata}/share/zoneinfo" + "CQLSH=${cqlsh-expansion}/bin/cqlsh-expansion" ]; - Env = [ "TZ=Etc/UTC" "TZDIR=${tzdata}/share/zoneinfo" "CQLSH=${cqlsh-expansion}/bin/cqlsh-expansion" "PYTHONUSERBASE=${cqlsh-expansion}" ]; }; }; diff --git a/nix/ocaml.nix b/nix/ocaml.nix index 0af21558283b..814d47754450 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -83,7 +83,8 @@ let # Also passes the version information to the executable. wrapMina = let commit_sha1 = inputs.self.sourceInfo.rev or ""; - commit_date = inputs.flockenzeit.lib.RFC-5322 inputs.self.sourceInfo.lastModified or 0; + commit_date = inputs.flockenzeit.lib.RFC-5322 + inputs.self.sourceInfo.lastModified or 0; in package: { deps ? [ pkgs.gnutar pkgs.gzip ], }: pkgs.runCommand "${package.name}-release" { From dcf9cca0cb18c2e20894fe04d73632df086c0813 Mon Sep 17 00:00:00 2001 From: Smorci Date: Thu, 14 Dec 2023 11:57:40 +0000 Subject: [PATCH 086/111] Add aws, base image from nix image, add authentication --- nix/docker.nix | 32 ++++++++++++++++--- .../delegation_verify/scripts/authenticate.sh | 14 ++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 src/app/delegation_verify/scripts/authenticate.sh diff --git a/nix/docker.nix b/nix/docker.nix index ec01c4fda943..c39ddbcd9a6e 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,7 +1,7 @@ { lib, dockerTools, buildEnv, ocamlPackages_mina, runCommand, dumb-init , coreutils, bashInteractive, python3, libp2p_helper, procps, postgresql, curl , jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit, tzdata -, cqlsh-expansion, python3Packages }: +, cqlsh-expansion, python3Packages, awscli, xorg }: let created = flockenzeit.lib.ISO-8601 currentTime; @@ -9,6 +9,15 @@ let mkdir = name: runCommand "mkdir-${name}" { } "mkdir -p $out${lib.escapeShellArg name}"; + nix-image = dockerTools.pullImage { + imageName = "lnl7/nix"; + imageDigest = + "sha256:9ba4c0a01d5153f741ac87364b2fd7a9c8b8b92600325f56d35da18421917e95"; + finalImageName = "lnl7-nix"; + finalImageTag = "latest"; + sha256 = "1zymy4rdwzbfhvcbpr1k3mr7gq08gkmrj33dg25ig0nbv9rka5si"; + }; + mina-build-config = stdenv.mkDerivation { pname = "mina-build-config"; version = "dev"; @@ -50,16 +59,24 @@ let }; mina-delegation-verify-init = runCommand "mina-delegation-verify-init" { } '' - mkdir -p $out - export HOME=$out export PYTHONUSERBASE=${cqlsh-expansion} - ${cqlsh-expansion}/bin/cqlsh-expansion.init - ''; + mina-delegation-verify-auth = stdenv.mkDerivation { + pname = "mina-delegation-verify-auth"; + version = "dev"; + src = ../src/app/delegation_verify/scripts; + outputs = [ "out" ]; + installPhase = '' + mkdir -p $out/bin + cp authenticate.sh $out/bin/authenticate.sh + chmod -R +x $out + ''; + }; + mkFullImage = name: packages: dockerTools.streamLayeredImage { name = "${name}-full"; @@ -95,10 +112,15 @@ in { mina-delegation-verify-image = dockerTools.streamLayeredImage { name = "mina-delegation-verify"; inherit created; + fromImage = nix-image; + maxLayers = 300; contents = [ ocamlPackages_mina.mina-delegation-verify.out cqlsh-expansion mina-delegation-verify-init + mina-delegation-verify-auth.out + awscli + jq coreutils bashInteractive ]; diff --git a/src/app/delegation_verify/scripts/authenticate.sh b/src/app/delegation_verify/scripts/authenticate.sh new file mode 100644 index 000000000000..049a0e71bc7f --- /dev/null +++ b/src/app/delegation_verify/scripts/authenticate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -x + +# The environment variables are set at the time of deployment in the init container +CREDENTIALS=$(aws sts assume-role --role-session-name $DELEGATION_VERIFY_AWS_ROLE_SESSION_NAME --role-arn $DELEGATION_VERIFY_AWS_ROLE_ARN) + +ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.Credentials.AccessKeyId') + +SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.Credentials.SecretAccessKey') + +SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.Credentials.SessionToken') + +echo "export AWS_ACCESS_KEY_ID=$ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$SECRET_ACCESS_KEY AWS_SESSION_TOKEN=$SESSION_TOKEN" > /var/mina-delegation-verify-auth/.env From cd8dd475eb0726bd64e7dcb9a7f3384447b8532c Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 15 Dec 2023 08:19:44 +0100 Subject: [PATCH 087/111] load from s3 if block not found in cassandra --- src/app/delegation_verify/README.md | 21 ++++++++++++++++++++ src/app/delegation_verify/submission.ml | 26 ++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index 97a316c525e3..eab0734b2614 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -147,6 +147,27 @@ Cassandra will be updated with the data. Any errors will also be logged in Cassandra and **will not** cause the verifier to exit with a non-zero exit code. +Fallback to AWS S3 +------------------ + +The Cassandra database (in Amazon Keyspaces) has a limitation +of 1MB per row. Consequently, blocks larger than this size will not be +present in the database. To address this, a fallback mechanism is implemented +to retrieve blocks from AWS S3 when they are not found in Cassandra. +This mode relies on `aws` cli tool that needs to be installed on the system. + +For this mechanism to function properly, the following environment variables must be set: + +* `AWS_CLI` - The path to the AWS CLI tool. If not set, the system defaults to using "/bin/aws". +* `AWS_ACCESS_KEY_ID` - AWS account's access key ID, required for AWS CLI tool to authenticate. +* `AWS_SECRET_ACCESS_KEY` - The secret key paired with access key ID. +* `AWS_S3_BUCKET` - The S3 bucket where submissions and blocks are stored. +* `NETWORK_NAME` - The network name identifier. + +The path to access appropriate block data in S3 is constructed based on these variables as follows: + +`AWS_S3_BUCKET`/`NETWORK_NAME`/blocks/.dat + Global options -------------- diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 95771dd4398e..42c074f92b60 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -230,6 +230,23 @@ module Cassandra = struct s :: l ) |> Deferred.return + let load_from_s3 ~block_hash = + let aws_cli = + match Sys.getenv "AWS_CLI" with + | Some path -> path + | _ -> "/bin/aws" + in + let bucket = Sys.getenv_exn "AWS_S3_BUCKET" in + let network = Sys.getenv_exn "NETWORK_NAME" in + let s3_path = sprintf "s3://%s/%s/blocks/%s.dat" bucket network block_hash in + + Process.run ~prog:aws_cli ~args:["s3"; "cp"; s3_path; "-"] () + >>= function + | Ok stdout -> + Deferred.Or_error.return (Some stdout) + | Error _ -> + Deferred.Or_error.return None + let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in let%bind block_data = @@ -239,7 +256,14 @@ module Cassandra = struct in match List.hd block_data with | None -> - Deferred.Or_error.error_string "Cassandra: Block not found" + (* If not found in Cassandra, try loading from S3 *) + begin + match%bind load_from_s3 ~block_hash with + | None -> + Deferred.Or_error.error_string "Block not found in Cassandra or S3" + | Some b -> + return b + end | Some b -> String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return From 8573bc7a6b7b46862285a66185910fd5aa7c1ef9 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Mon, 18 Dec 2023 17:47:35 +0100 Subject: [PATCH 088/111] applu code review suggestions --- src/app/delegation_verify/submission.ml | 34 ++++++++++--------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 42c074f92b60..d64ec77004a7 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -231,21 +231,19 @@ module Cassandra = struct |> Deferred.return let load_from_s3 ~block_hash = - let aws_cli = - match Sys.getenv "AWS_CLI" with - | Some path -> path - | _ -> "/bin/aws" + let aws_cli = Option.value ~default:"/bin/aws" @@ Sys.getenv "AWS_CLI" in + let s3_path = + let open Or_error.Let_syntax in + let%bind bucket = + Or_error.try_with (fun () -> Sys.getenv_exn "AWS_S3_BUCKET") + in + let%map network = + Or_error.try_with (fun () -> Sys.getenv_exn "NETWORK_NAME") + in + sprintf "s3://%s/%s/blocks/%s.dat" bucket network block_hash in - let bucket = Sys.getenv_exn "AWS_S3_BUCKET" in - let network = Sys.getenv_exn "NETWORK_NAME" in - let s3_path = sprintf "s3://%s/%s/blocks/%s.dat" bucket network block_hash in - - Process.run ~prog:aws_cli ~args:["s3"; "cp"; s3_path; "-"] () - >>= function - | Ok stdout -> - Deferred.Or_error.return (Some stdout) - | Error _ -> - Deferred.Or_error.return None + Deferred.Or_error.bind (return s3_path) ~f:(fun s3_path -> + Process.run ~prog:aws_cli ~args:[ "s3"; "cp"; s3_path; "-" ] () ) let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in @@ -257,13 +255,7 @@ module Cassandra = struct match List.hd block_data with | None -> (* If not found in Cassandra, try loading from S3 *) - begin - match%bind load_from_s3 ~block_hash with - | None -> - Deferred.Or_error.error_string "Block not found in Cassandra or S3" - | Some b -> - return b - end + load_from_s3 ~block_hash | Some b -> String.chop_prefix_exn b.raw_block ~prefix:"0x" |> Hex.Safe.of_hex |> Option.value_exn |> return From a94b34eb3d67d8b52182796da67a212c659937d0 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 20 Dec 2023 14:38:07 +0100 Subject: [PATCH 089/111] load_block_from_submission in Cassandra module --- .../delegation_verify/delegation_verify.ml | 14 ++++++++-- src/app/delegation_verify/submission.ml | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index c0a036ce16a8..586029758260 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -86,8 +86,14 @@ module Make_verifier (Source : Submission.Data_source) = struct let block_hash = Source.block_hash sub in if Known_blocks.is_known block_hash then () else - Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash - (Source.load_block src ~block_hash) + let load_block_action = + if Source.is_cassandra src then + Source.load_block_from_submission sub + else + Source.load_block src ~block_hash + in + Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash load_block_action + let verify ~validate (submission : Source.submission) = let open Deferred.Result.Let_syntax in @@ -161,6 +167,8 @@ let filesystem_command = let module V = Make_verifier (struct include Submission.Filesystem + let is_cassandra _ = false + let verify_blockchain_snarks = verify_blockchain_snarks let verify_transaction_snarks = verify_transaction_snarks @@ -193,6 +201,8 @@ let cassandra_command = let module V = Make_verifier (struct include Submission.Cassandra + let is_cassandra _ = true + let verify_blockchain_snarks = verify_blockchain_snarks let verify_transaction_snarks = verify_transaction_snarks diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index d64ec77004a7..0183b806d09c 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -15,6 +15,8 @@ module type Data_source = sig type submission + val is_cassandra : t -> bool + val submitted_at : submission -> string val block_hash : submission -> string @@ -26,6 +28,8 @@ module type Data_source = sig val load_submissions : t -> submission list Deferred.Or_error.t val load_block : block_hash:string -> t -> string Deferred.Or_error.t + + val load_block_from_submission : submission -> string Deferred.Or_error.t val verify_blockchain_snarks : (Mina_wire_types.Mina_state_protocol_state.Value.V2.t * Mina_base.Proof.t) @@ -115,6 +119,13 @@ module Filesystem = struct with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) + (* Dummy impl, not to be used in the context of Filesystem module. + It is only intended to fulfill the interface requirements of the + Data_source module signature for the Filesystem module *) + let load_block_from_submission submission = + let _ = submission in + Deferred.Or_error.return "dummy block data" + let output _ (_submission : submission) = function | Ok payload -> Output.display payload ; @@ -136,6 +147,7 @@ module Cassandra = struct ; block_hash : string ; submitted_at : string ; submitted_at_date : string + ; raw_block : string option [@default None] } type raw = @@ -148,6 +160,7 @@ module Cassandra = struct ; graphql_control_port : int option [@default None] ; submitted_at : string ; submitted_at_date : string + ; raw_block : string option [@default None] } [@@deriving yojson] @@ -170,6 +183,7 @@ module Cassandra = struct ; created_at = meta.created_at ; submitted_at_date = meta.submitted_at_date ; submitted_at = meta.submitted_at + ; raw_block = meta.raw_block } : submission ) @@ -212,6 +226,7 @@ module Cassandra = struct ; "submitter" ; "block_hash" ; "graphql_control_port" + ; "raw_block" ] ~where: (sprintf "%s AND submitted_at >= '%s' AND submitted_at < '%s'" @@ -245,6 +260,19 @@ module Cassandra = struct Deferred.Or_error.bind (return s3_path) ~f:(fun s3_path -> Process.run ~prog:aws_cli ~args:[ "s3"; "cp"; s3_path; "-" ] () ) + let load_block_from_submission (submission : submission) = + let open Deferred.Or_error.Let_syntax in + match submission.raw_block with + | None -> + (* If not found in Submission, try loading from S3 *) + load_from_s3 ~block_hash:submission.block_hash + | Some b -> + String.chop_prefix_exn b ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> return + + (* The 'blocks' table is no longer actively used in the Cassandra schema. + However, 'load_block' is retained for reference purposes and in case of + schema rollbacks or data migration needs. *) let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in let%bind block_data = From 452717f0788f6734eddd3393f37c03ab084dc288 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 20 Dec 2023 14:48:55 +0100 Subject: [PATCH 090/111] On update, purge heavy data (raw_block, snark_work) in Cassandra and mark row verified. --- src/app/delegation_verify/output.ml | 3 +++ src/app/delegation_verify/submission.ml | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml index e350715a9871..74a763110c74 100644 --- a/src/app/delegation_verify/output.ml +++ b/src/app/delegation_verify/output.ml @@ -26,6 +26,9 @@ let valid_payload_to_cassandra_updates (p : t) = ; ("parent", Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.parent) ; ( "state_hash" , Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.state_hash ) + ; ("raw_block", "NULL") + ; ("snark_work", "NULL") + ; ("verified", "true") ] let display valid_payload = diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index 0183b806d09c..b2313906b815 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -310,5 +310,9 @@ module Cassandra = struct (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) submission.submitted_at (Public_key.Compressed.to_base58_check submission.submitter) ) - [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) ] + [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) + ; ("raw_block", "NULL") + ; ("snark_work", "NULL") + ; ("verified", "true") + ] end From 2a243eda32c56a09b8ece35e9f961947349a37ed Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Mon, 8 Jan 2024 12:52:22 +0100 Subject: [PATCH 091/111] Add stateless_verifier dockerfile back --- .../Dockerfile-delegation-stateless-verifier | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 dockerfiles/Dockerfile-delegation-stateless-verifier diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier new file mode 100644 index 000000000000..f8c72cc8ddcb --- /dev/null +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -0,0 +1,58 @@ +################################################################################################# +# The "stateless verification build" Stage +# - builds stateless verification tool +# - should not include any data related to joining a specific network, only the node software itself +################################################################################################# +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:c810338e2c3973f7c674d0607048725917ce2be23b949c4bc9760c01122f884b AS builder + +# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI +ARG DUNE_PROFILE=devnet + +# branch to checkout on first clone (this will be the only availible branch in the container) +# can also be a tagged release +ARG MINA_BRANCH=sventimir/stateless-verification-tool + +# repo to checkout the branch from +ARG MINA_REPO=https://github.com/MinaProtocol/mina + +# location of repo used for pins and external package commits +ARG MINA_DIR=mina + +ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" + +# git will clone into an empty dir, but this also helps us set the workdir in advance +RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ + && git clone \ + -b "${MINA_BRANCH}" \ + --depth 1 \ + --shallow-submodules \ + --recurse-submodules \ + ${MINA_REPO} ${HOME}/${MINA_DIR} + +WORKDIR $HOME/${MINA_DIR} + +RUN git submodule sync && git submodule update --init --recursive + +RUN mkdir ${HOME}/app + +# HACK: build without special cpu features to allow more people to run delegation verification tool +# RUN ./scripts/zexe-standardize.sh + +RUN eval $(opam config env) \ + && dune build --profile=${DUNE_PROFILE} \ + src/app/delegation_verify/delegation_verify.exe \ + && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ + && rm -rf _build + +RUN rm -rf $HOME/${MINA_DIR} + +USER root + +# awscli and cqlsh-expansion are used by the delegation verification tool +RUN apt-get update && apt-get install -y python3 python3-pip +RUN pip3 install awscli +RUN pip3 install cqlsh-expansion + +RUN ln -s /usr/bin/aws /bin/aws + +ENTRYPOINT ["./delegation-verify"] \ No newline at end of file From eb429ea1d6fb0bb11ab551831205192248f2b8f8 Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 9 Jan 2024 14:47:23 +0000 Subject: [PATCH 092/111] Copy authentication script to docker image --- .../Dockerfile-delegation-stateless-verifier | 22 ++++++++++++++++--- .../delegation_verify/scripts/authenticate.sh | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index f8c72cc8ddcb..8795f3726c21 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -20,6 +20,12 @@ ARG MINA_DIR=mina ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" +RUN echo "Current user is: $(whoami)" + +RUN echo "Current dir is: $(pwd)" + +RUN echo "Path is: $PATH" + # git will clone into an empty dir, but this also helps us set the workdir in advance RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ && git clone \ @@ -33,6 +39,7 @@ WORKDIR $HOME/${MINA_DIR} RUN git submodule sync && git submodule update --init --recursive + RUN mkdir ${HOME}/app # HACK: build without special cpu features to allow more people to run delegation verification tool @@ -44,10 +51,19 @@ RUN eval $(opam config env) \ && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ && rm -rf _build -RUN rm -rf $HOME/${MINA_DIR} - USER root +# copy binary to /bin +RUN cp ./delegation-verify /bin/delegation-verify + +# add authenticate.sh to image +RUN cp src/app/delegation_verify/scripts/authenticate.sh /bin/authenticate.sh + +# make binary and script executable +RUN chmod +x /bin/authenticate.sh /bin/delegation-verify + +RUN rm -rf $HOME/${MINA_DIR} + # awscli and cqlsh-expansion are used by the delegation verification tool RUN apt-get update && apt-get install -y python3 python3-pip RUN pip3 install awscli @@ -55,4 +71,4 @@ RUN pip3 install cqlsh-expansion RUN ln -s /usr/bin/aws /bin/aws -ENTRYPOINT ["./delegation-verify"] \ No newline at end of file +ENTRYPOINT ["./delegation-verify"] diff --git a/src/app/delegation_verify/scripts/authenticate.sh b/src/app/delegation_verify/scripts/authenticate.sh index 049a0e71bc7f..331a84fdafbf 100644 --- a/src/app/delegation_verify/scripts/authenticate.sh +++ b/src/app/delegation_verify/scripts/authenticate.sh @@ -3,7 +3,7 @@ set -x # The environment variables are set at the time of deployment in the init container -CREDENTIALS=$(aws sts assume-role --role-session-name $DELEGATION_VERIFY_AWS_ROLE_SESSION_NAME --role-arn $DELEGATION_VERIFY_AWS_ROLE_ARN) +CREDENTIALS=$(aws sts assume-role --role-session-name $AWS_ROLE_SESSION_NAME --role-arn $AWS_ROLE_ARN) ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.Credentials.AccessKeyId') From 06d4a2aa8b078995a29bb21f14b3ac6745c63c12 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 12 Jan 2024 18:22:25 +0100 Subject: [PATCH 093/111] fix entrypoint --- dockerfiles/Dockerfile-delegation-stateless-verifier | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 8795f3726c21..c48697b2e55b 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -70,5 +70,7 @@ RUN pip3 install awscli RUN pip3 install cqlsh-expansion RUN ln -s /usr/bin/aws /bin/aws +RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh +RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion -ENTRYPOINT ["./delegation-verify"] +ENTRYPOINT ["/bin/delegation-verify"] From 5d12b04ae5472e8fb6205aad7869fc16a0fb5577 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 12 Jan 2024 19:11:21 +0100 Subject: [PATCH 094/111] run cqlsh-expansion.init --- dockerfiles/Dockerfile-delegation-stateless-verifier | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index c48697b2e55b..947b38c7aadc 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -72,5 +72,6 @@ RUN pip3 install cqlsh-expansion RUN ln -s /usr/bin/aws /bin/aws RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion +RUN /usr/local/bin/cqlsh-expansion.init ENTRYPOINT ["/bin/delegation-verify"] From 028fe9eb57434deac9822cd339eace07d4f002e3 Mon Sep 17 00:00:00 2001 From: Smorci Date: Mon, 15 Jan 2024 11:23:48 +0000 Subject: [PATCH 095/111] Multi stage docker delegation verify --- .../Dockerfile-delegation-stateless-verifier | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 947b38c7aadc..d0332843e177 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -20,12 +20,6 @@ ARG MINA_DIR=mina ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" -RUN echo "Current user is: $(whoami)" - -RUN echo "Current dir is: $(pwd)" - -RUN echo "Path is: $PATH" - # git will clone into an empty dir, but this also helps us set the workdir in advance RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ && git clone \ @@ -39,7 +33,6 @@ WORKDIR $HOME/${MINA_DIR} RUN git submodule sync && git submodule update --init --recursive - RUN mkdir ${HOME}/app # HACK: build without special cpu features to allow more people to run delegation verification tool @@ -59,19 +52,25 @@ RUN cp ./delegation-verify /bin/delegation-verify # add authenticate.sh to image RUN cp src/app/delegation_verify/scripts/authenticate.sh /bin/authenticate.sh -# make binary and script executable -RUN chmod +x /bin/authenticate.sh /bin/delegation-verify - -RUN rm -rf $HOME/${MINA_DIR} +# Runtime image +FROM ubuntu:latest # awscli and cqlsh-expansion are used by the delegation verification tool -RUN apt-get update && apt-get install -y python3 python3-pip +RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 libssl-dev RUN pip3 install awscli RUN pip3 install cqlsh-expansion +# Copy resources from builder to runtime image +COPY --from=builder /bin/delegation-verify /bin/delegation-verify +COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh + +# Make symlinks RUN ln -s /usr/bin/aws /bin/aws RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion RUN /usr/local/bin/cqlsh-expansion.init +# make binary and script executable +RUN chmod +x /bin/authenticate.sh /bin/delegation-verify + ENTRYPOINT ["/bin/delegation-verify"] From c7affcb4c478249e865a6cf73451672526c952fa Mon Sep 17 00:00:00 2001 From: Smorci Date: Mon, 15 Jan 2024 15:22:06 +0000 Subject: [PATCH 096/111] Add C libraries to delegation verify docker image --- .../Dockerfile-delegation-stateless-verifier | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index d0332843e177..3dbd646d05f4 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -55,14 +55,34 @@ RUN cp src/app/delegation_verify/scripts/authenticate.sh /bin/authenticate.sh # Runtime image FROM ubuntu:latest +# Copy resources from builder to runtime image +COPY --from=builder /bin/delegation-verify /bin/delegation-verify +COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh + # awscli and cqlsh-expansion are used by the delegation verification tool -RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 libssl-dev +RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget RUN pip3 install awscli RUN pip3 install cqlsh-expansion -# Copy resources from builder to runtime image -COPY --from=builder /bin/delegation-verify /bin/delegation-verify -COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh +# Install libssl1.1.1b (not in apt) +RUN wget https://www.openssl.org/source/openssl-1.1.1b.tar.gz +RUN mkdir /opt/openssl +RUN tar xfvz openssl-1.1.1b.tar.gz --directory /opt/openssl +RUN rm openssl-1.1.1b.tar.gz + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/openssl/lib" +ENV PATH="$PATH:/opt/openssl/bin" + +RUN cd /opt/openssl/openssl-1.1.1b && ./config --prefix=/opt/openssl --openssldir=/opt/openssl/ssl +RUN cd /opt/openssl/openssl-1.1.1b && make && make install + +# Rename openssl old binary +RUN mv /usr/bin/openssl /usr/bin/openssl.old + +# Install libffi7 +RUN wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb +RUN dpkg -i libffi7_3.3-4_amd64.deb +RUN rm libffi7_3.3-4_amd64.deb # Make symlinks RUN ln -s /usr/bin/aws /bin/aws From 8f771a8af6c49748b5f6a7156705521af47ba686 Mon Sep 17 00:00:00 2001 From: Smorci Date: Mon, 15 Jan 2024 16:26:00 +0000 Subject: [PATCH 097/111] Configure localtime in docker image for delegation verify --- dockerfiles/Dockerfile-delegation-stateless-verifier | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 3dbd646d05f4..45e17b7cc7a1 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -93,4 +93,9 @@ RUN /usr/local/bin/cqlsh-expansion.init # make binary and script executable RUN chmod +x /bin/authenticate.sh /bin/delegation-verify +# set up timezone +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ="Etc/UTC" +RUN apt install tzdata + ENTRYPOINT ["/bin/delegation-verify"] From 36463f73ae9da3cd766af5e91fd47d7673c48dde Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 16 Jan 2024 14:39:59 +0000 Subject: [PATCH 098/111] Remove nix changes - moved to branch smorci/nix-cqlsh-delegation-verify for safekeeping --- flake.nix | 5 ++--- nix/README.md | 18 ---------------- nix/docker.nix | 58 +------------------------------------------------- nix/ocaml.nix | 19 ----------------- nix/python.nix | 46 --------------------------------------- 5 files changed, 3 insertions(+), 143 deletions(-) delete mode 100644 nix/python.nix diff --git a/flake.nix b/flake.nix index 725ac2846c90..1bae798d9b02 100644 --- a/flake.nix +++ b/flake.nix @@ -69,7 +69,6 @@ rust = import ./nix/rust.nix; go = import ./nix/go.nix; javascript = import ./nix/javascript.nix; - python = import ./nix/python.nix; ocaml = final: prev: { ocamlPackages_mina = requireSubmodules (import ./nix/ocaml.nix { inherit inputs; @@ -288,12 +287,12 @@ # Main user-facing binaries. packages = rec { inherit (ocamlPackages) - mina mina_tests mina-ocaml-format test_executive mina-delegation-verify; + mina mina_tests mina-ocaml-format test_executive; inherit (pkgs) libp2p_helper kimchi_bindings_stubs snarky_js leaderboard validation trace-tool zkapp-cli; inherit (dockerImages) - mina-image-slim mina-image-full mina-archive-image-full mina-delegation-verify-image; + mina-image-slim mina-image-full mina-archive-image-full; mina-deb = debianPackages.mina; default = mina; }; diff --git a/nix/README.md b/nix/README.md index 44e143ca245c..4903846d2fc4 100644 --- a/nix/README.md +++ b/nix/README.md @@ -182,23 +182,6 @@ Alternatively, you can build the entirety of Mina inside the Nix sandbox fully automatically: run `nix build mina` if you're using flakes (or `nix-build` otherwise). You can find the resulting executable at `result/bin/mina`. -To build the available packages use `nix build mina#` where `` is from the following list. - -Available packages: -- **mina** - mina daemon executable -- **mina-deb** - mina debian package -- **mina_tests** - mina unit tests -- **mina-ocaml-format** - checks if the code is formatted properly -- **mina-delegation-verify** - [stateless verification tool](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/delegation_verify) for Delegation Program -- **test_executive** - [test executive](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/test_executive) executable -- **libp2p_helper** - helper go module for unit tests -- **kimchi_bindings_stubs** -- **snarky_js** - builds snarkyjs npm package -- **leaderboard** - builds the [leaderboard](https://github.com/MinaProtocol/mina/tree/berkeley/frontend/leaderboard) yarn package -- **validation** - builds the [validation](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/validation) app -- **trace-tool** - builds the [trace-tool](https://github.com/MinaProtocol/mina/tree/berkeley/src/app/trace-tool) -- **zkapp-cli** - builds the [zkapp-cli](https://github.com/o1-labs/zkapp-cli/) - ### "Impure" development shell TL;DR: @@ -245,7 +228,6 @@ Available images: - **mina-image-full** - full Mina daemon image with useful tools, such as coreutils, fake init, jq, etc. - **mina-image-slim** - has Mina daemon only - **mina-archive-image-full** - full Mina archive image with useful tools -- **mina-delegation-verify-image** - stateless verification tool for Delegation Program ### Debian package diff --git a/nix/docker.nix b/nix/docker.nix index c39ddbcd9a6e..e2f55a8dd4b6 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -1,23 +1,12 @@ { lib, dockerTools, buildEnv, ocamlPackages_mina, runCommand, dumb-init , coreutils, bashInteractive, python3, libp2p_helper, procps, postgresql, curl -, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit, tzdata -, cqlsh-expansion, python3Packages, awscli, xorg }: +, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit }: let - created = flockenzeit.lib.ISO-8601 currentTime; mkdir = name: runCommand "mkdir-${name}" { } "mkdir -p $out${lib.escapeShellArg name}"; - nix-image = dockerTools.pullImage { - imageName = "lnl7/nix"; - imageDigest = - "sha256:9ba4c0a01d5153f741ac87364b2fd7a9c8b8b92600325f56d35da18421917e95"; - finalImageName = "lnl7-nix"; - finalImageTag = "latest"; - sha256 = "1zymy4rdwzbfhvcbpr1k3mr7gq08gkmrj33dg25ig0nbv9rka5si"; - }; - mina-build-config = stdenv.mkDerivation { pname = "mina-build-config"; version = "dev"; @@ -58,25 +47,6 @@ let ''; }; - mina-delegation-verify-init = runCommand "mina-delegation-verify-init" { } '' - mkdir -p $out - export HOME=$out - export PYTHONUSERBASE=${cqlsh-expansion} - ${cqlsh-expansion}/bin/cqlsh-expansion.init - ''; - - mina-delegation-verify-auth = stdenv.mkDerivation { - pname = "mina-delegation-verify-auth"; - version = "dev"; - src = ../src/app/delegation_verify/scripts; - outputs = [ "out" ]; - installPhase = '' - mkdir -p $out/bin - cp authenticate.sh $out/bin/authenticate.sh - chmod -R +x $out - ''; - }; - mkFullImage = name: packages: dockerTools.streamLayeredImage { name = "${name}-full"; @@ -108,32 +78,6 @@ in { inherit created; contents = [ ocamlPackages_mina.mina.out ]; }; - - mina-delegation-verify-image = dockerTools.streamLayeredImage { - name = "mina-delegation-verify"; - inherit created; - fromImage = nix-image; - maxLayers = 300; - contents = [ - ocamlPackages_mina.mina-delegation-verify.out - cqlsh-expansion - mina-delegation-verify-init - mina-delegation-verify-auth.out - awscli - jq - coreutils - bashInteractive - ]; - config = { - cmd = [ "bash" ]; - Env = [ - "TZ=Etc/UTC" - "TZDIR=${tzdata}/share/zoneinfo" - "CQLSH=${cqlsh-expansion}/bin/cqlsh-expansion" - ]; - }; - }; - mina-image-full = mkFullImage "mina" (with ocamlPackages_mina; [ mina-build-config mina-daemon-scripts diff --git a/nix/ocaml.nix b/nix/ocaml.nix index 814d47754450..eddac9a6ebae 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -322,25 +322,6 @@ let ''; }); - # Stateless verification tool - mina-delegation-verify-dev = self.mina-dev.overrideAttrs (_: { - pname = "mina-delegation-verify"; - version = "dev"; - outputs = [ "out" ]; - - buildPhase = '' - dune build --display=short --profile=devnet \ - src/app/delegation_verify/delegation_verify.exe - ''; - - installPhase = '' - mkdir -p $out/bin - mv _build/default/src/app/delegation_verify/delegation_verify.exe $out/bin/delegation-verify - ''; - }); - - mina-delegation-verify = wrapMina self.mina-delegation-verify-dev { }; - # Integration test executive test_executive-dev = self.mina-dev.overrideAttrs (oa: { pname = "mina-test_executive"; diff --git a/nix/python.nix b/nix/python.nix deleted file mode 100644 index abd8b542181c..000000000000 --- a/nix/python.nix +++ /dev/null @@ -1,46 +0,0 @@ -# An overlay defining Python packages -final: prev: { - - cassandra-sigv4 = final.python3Packages.buildPythonPackage rec { - pname = "cassandra-sigv4"; - version = "4.0.2"; - pyproject = true; - src = final.python3Packages.fetchPypi { - inherit pname version; - hash = "sha256-d/9hI6uUHFCnCuUXed5ae74oQ9lrgMRmrP/fJil0ipg="; - }; - nativeBuildInputs = with prev.python3Packages; [ - setuptools-scm - six - cassandra-driver - boto3 - mock - ]; - doCheck = false; - }; - - cqlsh-expansion = final.python3Packages.buildPythonPackage rec { - pname = "cqlsh-expansion"; - version = "0.9.6"; - pyproject = true; - src = final.python3Packages.fetchPypi { - inherit pname version; - hash = "sha256-ea5ew7aF19JcdzoMgLrvlj0GC/wLfbIsABLKRxcT6DQ="; - }; - buildInputs = with final.python3Packages; [ - cassandra-driver - six - setuptools-scm - final.cassandra-sigv4 - boto3 - ]; - pythonPath = with final.python3Packages; [ - cassandra-driver - six - final.cassandra-sigv4 - boto3 - ]; - permitUserSite = true; - doCheck = false; - }; -} From 6722c242d630e8bb7ad0b765da24134b60600839 Mon Sep 17 00:00:00 2001 From: Smorci Date: Tue, 16 Jan 2024 14:43:07 +0000 Subject: [PATCH 099/111] Change build instructions on README --- src/app/delegation_verify/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md index eab0734b2614..ff147fe9f61c 100644 --- a/src/app/delegation_verify/README.md +++ b/src/app/delegation_verify/README.md @@ -186,4 +186,8 @@ Some command-line options work with both modes of operation: Build ----- -The executable and docker image can be built with Nix. For further instructions, consult the Mina [Nix documentation](https://github.com/MinaProtocol/mina/tree/berkeley/nix) +To build the docker image go to the root of the repositoryand run + +`docker build -t -f dockerfiles/Dockerfile-delegation-stateless-verifier .` + +where `` is your desired image name. From 4acffe19e38861df6f28b39a1e3c5493f041dd87 Mon Sep 17 00:00:00 2001 From: Smorci Date: Wed, 17 Jan 2024 12:00:03 +0000 Subject: [PATCH 100/111] Change HOME variable in delegation verify docker image --- dockerfiles/Dockerfile-delegation-stateless-verifier | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 45e17b7cc7a1..3315d0f6276b 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -98,4 +98,7 @@ ENV DEBIAN_FRONTEND="noninteractive" ENV TZ="Etc/UTC" RUN apt install tzdata +# set home to root dir +ENV HOME="/root" + ENTRYPOINT ["/bin/delegation-verify"] From ba3c89ae1f3c0ccf1d3047c960c0c434e205e6e5 Mon Sep 17 00:00:00 2001 From: Smorci Date: Fri, 19 Jan 2024 10:41:11 +0000 Subject: [PATCH 101/111] Add awk and dnsutils to delegation verify docker image --- dockerfiles/Dockerfile-delegation-stateless-verifier | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 3315d0f6276b..2513926435a1 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -60,7 +60,7 @@ COPY --from=builder /bin/delegation-verify /bin/delegation-verify COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh # awscli and cqlsh-expansion are used by the delegation verification tool -RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget +RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget dnsutils gawk RUN pip3 install awscli RUN pip3 install cqlsh-expansion From c82ed1bc18f51265e2ac1dfdebaedbb10d7c398f Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 23 Jan 2024 09:20:53 +0100 Subject: [PATCH 102/111] fix /bin/aws link --- dockerfiles/Dockerfile-delegation-stateless-verifier | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index 2513926435a1..f1b5f8311e67 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -85,7 +85,7 @@ RUN dpkg -i libffi7_3.3-4_amd64.deb RUN rm libffi7_3.3-4_amd64.deb # Make symlinks -RUN ln -s /usr/bin/aws /bin/aws +RUN ln -s /usr/local/bin/aws /bin/aws RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion RUN /usr/local/bin/cqlsh-expansion.init From 9a9bc3d27e00b5920a38d27290fdfde837449be6 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 23 Jan 2024 11:54:15 +0100 Subject: [PATCH 103/111] add pytz --- dockerfiles/Dockerfile-delegation-stateless-verifier | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier index f1b5f8311e67..381d6f3631cf 100644 --- a/dockerfiles/Dockerfile-delegation-stateless-verifier +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -63,6 +63,7 @@ COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget dnsutils gawk RUN pip3 install awscli RUN pip3 install cqlsh-expansion +RUN pip3 install pytz # Install libssl1.1.1b (not in apt) RUN wget https://www.openssl.org/source/openssl-1.1.1b.tar.gz From ec25f6f7e2ff5e6aebbdf4aa7b4cea4d4ed88b68 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:26:45 +0000 Subject: [PATCH 104/111] Update input type --- src/app/delegation_verify/submission.ml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml index b2313906b815..80af3f12af4f 100644 --- a/src/app/delegation_verify/submission.ml +++ b/src/app/delegation_verify/submission.ml @@ -16,7 +16,7 @@ module type Data_source = sig type submission val is_cassandra : t -> bool - + val submitted_at : submission -> string val block_hash : submission -> string @@ -28,7 +28,7 @@ module type Data_source = sig val load_submissions : t -> submission list Deferred.Or_error.t val load_block : block_hash:string -> t -> string Deferred.Or_error.t - + val load_block_from_submission : submission -> string Deferred.Or_error.t val verify_blockchain_snarks : @@ -55,7 +55,7 @@ module Filesystem = struct } type raw = - { submitted_at : string + { created_at : string ; peer_id : string ; snark_work : string option [@default None] ; remote_addr : string @@ -81,7 +81,7 @@ module Filesystem = struct { submitter ; snark_work ; block_hash = meta.block_hash - ; submitted_at = meta.submitted_at + ; submitted_at = meta.created_at } let submitted_at ({ submitted_at; _ } : submission) = submitted_at @@ -119,8 +119,8 @@ module Filesystem = struct with _ -> Error (Error.of_string "Fail to load block") ) |> Ivar.fill ivar ) - (* Dummy impl, not to be used in the context of Filesystem module. - It is only intended to fulfill the interface requirements of the + (* Dummy impl, not to be used in the context of Filesystem module. + It is only intended to fulfill the interface requirements of the Data_source module signature for the Filesystem module *) let load_block_from_submission submission = let _ = submission in @@ -259,19 +259,19 @@ module Cassandra = struct in Deferred.Or_error.bind (return s3_path) ~f:(fun s3_path -> Process.run ~prog:aws_cli ~args:[ "s3"; "cp"; s3_path; "-" ] () ) - + let load_block_from_submission (submission : submission) = let open Deferred.Or_error.Let_syntax in match submission.raw_block with | None -> - (* If not found in Submission, try loading from S3 *) - load_from_s3 ~block_hash:submission.block_hash + (* If not found in Submission, try loading from S3 *) + load_from_s3 ~block_hash:submission.block_hash | Some b -> String.chop_prefix_exn b ~prefix:"0x" - |> Hex.Safe.of_hex |> Option.value_exn |> return + |> Hex.Safe.of_hex |> Option.value_exn |> return (* The 'blocks' table is no longer actively used in the Cassandra schema. - However, 'load_block' is retained for reference purposes and in case of + However, 'load_block' is retained for reference purposes and in case of schema rollbacks or data migration needs. *) let load_block ~block_hash { conf; _ } = let open Deferred.Or_error.Let_syntax in @@ -310,7 +310,7 @@ module Cassandra = struct (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) submission.submitted_at (Public_key.Compressed.to_base58_check submission.submitter) ) - [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) + [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) ; ("raw_block", "NULL") ; ("snark_work", "NULL") ; ("verified", "true") From 75b1cab6b045683defdfeac7767ab482898469be Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:00:06 +0000 Subject: [PATCH 105/111] Reformat --- src/app/delegation_verify/delegation_verify.ml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml index 586029758260..6549979b5d12 100644 --- a/src/app/delegation_verify/delegation_verify.ml +++ b/src/app/delegation_verify/delegation_verify.ml @@ -87,13 +87,11 @@ module Make_verifier (Source : Submission.Data_source) = struct if Known_blocks.is_known block_hash then () else let load_block_action = - if Source.is_cassandra src then - Source.load_block_from_submission sub - else - Source.load_block src ~block_hash + if Source.is_cassandra src then Source.load_block_from_submission sub + else Source.load_block src ~block_hash in - Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash load_block_action - + Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash + load_block_action let verify ~validate (submission : Source.submission) = let open Deferred.Result.Let_syntax in From 21f5e5caa0f2ceff9eb111cb895b548d7e6ad40f Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 20 Feb 2024 11:45:09 +0100 Subject: [PATCH 106/111] Revert changes to nix configuration. --- nix/docker.nix | 58 +++++++++++++++++++-------------------- nix/libp2p_helper.json | 2 +- nix/ocaml.nix | 3 +- scripts/release-docker.sh | 6 +--- 4 files changed, 31 insertions(+), 38 deletions(-) diff --git a/nix/docker.nix b/nix/docker.nix index e2f55a8dd4b6..748c23369522 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -47,30 +47,29 @@ let ''; }; - mkFullImage = name: packages: - dockerTools.streamLayeredImage { - name = "${name}-full"; - inherit created; - contents = [ - dumb-init - coreutils - bashInteractive - python3 - libp2p_helper - procps - curl - jq - ] ++ packages; - extraCommands = '' - mkdir root tmp - chmod 777 tmp - ''; - config = { - env = [ "MINA_TIME_OFFSET=0" ]; - WorkingDir = "/root"; - cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; - }; + mkFullImage = name: packages: dockerTools.streamLayeredImage { + name = "${name}-full"; + inherit created; + contents = [ + dumb-init + coreutils + bashInteractive + python3 + libp2p_helper + procps + curl + jq + ] ++ packages; + extraCommands = '' + mkdir root tmp + chmod 777 tmp + ''; + config = { + env = [ "MINA_TIME_OFFSET=0" ]; + WorkingDir = "/root"; + cmd = [ "/bin/dumb-init" "/entrypoint.sh" ]; }; + }; in { mina-image-slim = dockerTools.streamLayeredImage { @@ -86,12 +85,11 @@ in { mina.mainnet mina.genesis ]); - mina-archive-image-full = mkFullImage "mina-archive" - (with ocamlPackages_mina; [ - mina-archive-scripts - gnutar - gzip + mina-archive-image-full = mkFullImage "mina-archive" (with ocamlPackages_mina; [ + mina-archive-scripts + gnutar + gzip - mina.archive - ]); + mina.archive + ]); } diff --git a/nix/libp2p_helper.json b/nix/libp2p_helper.json index 650889eddc1c..7a0a9b7e34a7 100644 --- a/nix/libp2p_helper.json +++ b/nix/libp2p_helper.json @@ -1 +1 @@ -{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} +{"go.mod":"d5de7e35a76f5c9ce7d6c98f0da39c763961e77b8c94761b1e89ab4bdfdc2a97","go.sum":"586fd920114d3875ec3e1d739921d77d30ad8e2f297b67781ca41d25a81b65a9","vendorSha256":"sha256-vyKrKi5bqm8Mf2rUOojSY0IXHcuNpcVNvd1Iu1RBxDo="} \ No newline at end of file diff --git a/nix/ocaml.nix b/nix/ocaml.nix index ce9e2ba484d3..1233edb92739 100644 --- a/nix/ocaml.nix +++ b/nix/ocaml.nix @@ -83,8 +83,7 @@ let # Also passes the version information to the executable. wrapMina = let commit_sha1 = inputs.self.sourceInfo.rev or ""; - commit_date = inputs.flockenzeit.lib.RFC-5322 - inputs.self.sourceInfo.lastModified or 0; + commit_date = inputs.flockenzeit.lib.RFC-5322 inputs.self.sourceInfo.lastModified or 0; in package: { deps ? [ pkgs.gnutar pkgs.gzip ], }: pkgs.runCommand "${package.name}-release" { diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 1ef1fcdad1e0..a07d1ebfae8f 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -10,7 +10,7 @@ set +x CLEAR='\033[0m' RED='\033[0;31m' # Array of valid service names -VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-test-suite' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'itn-orchestrator' 'delegation-verify') +VALID_SERVICES=('mina-archive', 'mina-daemon' 'mina-rosetta' 'mina-test-suite' 'mina-test-executive' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'itn-orchestrator') function usage() { if [[ -n "$1" ]]; then @@ -139,10 +139,6 @@ mina-test-suite) DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-test-suite" DOCKER_CONTEXT="dockerfiles/" ;; - -delegation-verify) - DOCKERFILE_PATH="dockerfiles/Dockerfile-delegation-stateless-verifier" - ;; esac From 4511e76ac24ad3f27230d38f41482b828017678a Mon Sep 17 00:00:00 2001 From: Sventimir Date: Tue, 20 Feb 2024 17:48:07 +0100 Subject: [PATCH 107/111] Rename a record field for clarity. --- src/app/delegation_verify/cassandra.ml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index d32b112ef477..3d76c261802e 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -3,7 +3,7 @@ open Core type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or -type connection_conf = { hostname : string; port : int; ssl : bool } +type connection_conf = { hostname : string; port : int; use_ssl : bool } type credentials = { username : string; password : string } @@ -18,12 +18,12 @@ let make_conn_conf () : connection_conf option = let open Option.Let_syntax in let%bind hostname = Sys.getenv "CASSANDRA_HOST" in let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in - let%map ssl = + let%map use_ssl = Sys.getenv "CASSANDRA_USE_SSL" |> Option.map ~f:(List.mem ~equal:String.equal [ "1"; "TRUE"; "true"; "YES"; "yes" ]) in - { hostname; port; ssl } + { hostname; port; use_ssl } let make_cred_conf () : credentials option = let open Option.Let_syntax in @@ -45,8 +45,8 @@ let query ~conf q = let args = optional conf.credentials ~f:(fun { username; password } -> [ "--username"; username; "--password"; password ] ) - @ optional conf.connection ~f:(fun { hostname; port; ssl } -> - (if ssl then [ "--ssl" ] else []) @ [ hostname; Int.to_string port ] ) + @ optional conf.connection ~f:(fun { hostname; port; use_ssl } -> + (if use_ssl then [ "--ssl" ] else []) @ [ hostname; Int.to_string port ] ) in Process.run_lines ~prog:conf.executable ~stdin:q ~args () From e1a16ef505f8b5b69fa7582f2c2e49641c5db7dd Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:27:41 +0000 Subject: [PATCH 108/111] Reformat --- src/app/delegation_verify/cassandra.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml index 3d76c261802e..bc4296e881e1 100644 --- a/src/app/delegation_verify/cassandra.ml +++ b/src/app/delegation_verify/cassandra.ml @@ -46,7 +46,8 @@ let query ~conf q = optional conf.credentials ~f:(fun { username; password } -> [ "--username"; username; "--password"; password ] ) @ optional conf.connection ~f:(fun { hostname; port; use_ssl } -> - (if use_ssl then [ "--ssl" ] else []) @ [ hostname; Int.to_string port ] ) + (if use_ssl then [ "--ssl" ] else []) + @ [ hostname; Int.to_string port ] ) in Process.run_lines ~prog:conf.executable ~stdin:q ~args () From 0850740f25f42b6a68fa12d9d47897b972b5581f Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:50:56 +0000 Subject: [PATCH 109/111] Revert README --- nix/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/README.md b/nix/README.md index 4903846d2fc4..9163d282a3be 100644 --- a/nix/README.md +++ b/nix/README.md @@ -209,7 +209,8 @@ branches, or otherwise changing the dependency tree of Mina. TL;DR: ``` -$(nix build mina#mina-image-full --print-out-paths ) | docker load +$(nix build mina#mina-image-full) | docker load +# Also available: mina-image-slim, mina-archive-image-full ``` Since a "pure" build can happen entirely inside the Nix sandbox, we can use its @@ -225,9 +226,8 @@ registry at us-west2-docker.pkg.dev/o1labs-192920/nix-containers/mina-image-full:develop` . Available images: -- **mina-image-full** - full Mina daemon image with useful tools, such as coreutils, fake init, jq, etc. -- **mina-image-slim** - has Mina daemon only -- **mina-archive-image-full** - full Mina archive image with useful tools +The `slim` image only has the Mina daemon itself, whereas `full` images also +contain many useful tools, such as coreutils, fake init, jq, etc. ### Debian package From 26a8271063d349629cfb660d8f3ab024bf7029de Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:52:51 +0000 Subject: [PATCH 110/111] ReadMe --- nix/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/README.md b/nix/README.md index 9163d282a3be..08d058b31a5f 100644 --- a/nix/README.md +++ b/nix/README.md @@ -225,7 +225,6 @@ registry at `docker run --rm -it us-west2-docker.pkg.dev/o1labs-192920/nix-containers/mina-image-full:develop` . -Available images: The `slim` image only has the Mina daemon itself, whereas `full` images also contain many useful tools, such as coreutils, fake init, jq, etc. From deed738634454a1b4cfd6f9b940d6bad2b376298 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:56:34 +0000 Subject: [PATCH 111/111] readMe --- nix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/README.md b/nix/README.md index 08d058b31a5f..725f98976ac2 100644 --- a/nix/README.md +++ b/nix/README.md @@ -601,4 +601,4 @@ Before running any `dune` commands. Alternatively, you can just run your commands inside `nix develop --ignore-environment mina`, which unsets all the outside environment variables, -resulting in a more reproducible but less convenient environment. +resulting in a more reproducible but less convenient environment. \ No newline at end of file