diff --git a/dockerfiles/Dockerfile-delegation-stateless-verifier b/dockerfiles/Dockerfile-delegation-stateless-verifier new file mode 100644 index 000000000000..381d6f3631cf --- /dev/null +++ b/dockerfiles/Dockerfile-delegation-stateless-verifier @@ -0,0 +1,105 @@ +################################################################################################# +# The "stateless verification build" Stage +# - builds stateless verification tool +# - should not include any data related to joining a specific network, only the node software itself +################################################################################################# +FROM gcr.io/o1labs-192920/mina-toolchain@sha256:c810338e2c3973f7c674d0607048725917ce2be23b949c4bc9760c01122f884b AS builder + +# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI +ARG DUNE_PROFILE=devnet + +# branch to checkout on first clone (this will be the only availible branch in the container) +# can also be a tagged release +ARG MINA_BRANCH=sventimir/stateless-verification-tool + +# repo to checkout the branch from +ARG MINA_REPO=https://github.com/MinaProtocol/mina + +# location of repo used for pins and external package commits +ARG MINA_DIR=mina + +ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" + +# git will clone into an empty dir, but this also helps us set the workdir in advance +RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ + && git clone \ + -b "${MINA_BRANCH}" \ + --depth 1 \ + --shallow-submodules \ + --recurse-submodules \ + ${MINA_REPO} ${HOME}/${MINA_DIR} + +WORKDIR $HOME/${MINA_DIR} + +RUN git submodule sync && git submodule update --init --recursive + +RUN mkdir ${HOME}/app + +# HACK: build without special cpu features to allow more people to run delegation verification tool +# RUN ./scripts/zexe-standardize.sh + +RUN eval $(opam config env) \ + && dune build --profile=${DUNE_PROFILE} \ + src/app/delegation_verify/delegation_verify.exe \ + && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \ + && rm -rf _build + +USER root + +# copy binary to /bin +RUN cp ./delegation-verify /bin/delegation-verify + +# add authenticate.sh to image +RUN cp src/app/delegation_verify/scripts/authenticate.sh /bin/authenticate.sh + +# Runtime image +FROM ubuntu:latest + +# Copy resources from builder to runtime image +COPY --from=builder /bin/delegation-verify /bin/delegation-verify +COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh + +# awscli and cqlsh-expansion are used by the delegation verification tool +RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget dnsutils gawk +RUN pip3 install awscli +RUN pip3 install cqlsh-expansion +RUN pip3 install pytz + +# Install libssl1.1.1b (not in apt) +RUN wget https://www.openssl.org/source/openssl-1.1.1b.tar.gz +RUN mkdir /opt/openssl +RUN tar xfvz openssl-1.1.1b.tar.gz --directory /opt/openssl +RUN rm openssl-1.1.1b.tar.gz + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/openssl/lib" +ENV PATH="$PATH:/opt/openssl/bin" + +RUN cd /opt/openssl/openssl-1.1.1b && ./config --prefix=/opt/openssl --openssldir=/opt/openssl/ssl +RUN cd /opt/openssl/openssl-1.1.1b && make && make install + +# Rename openssl old binary +RUN mv /usr/bin/openssl /usr/bin/openssl.old + +# Install libffi7 +RUN wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb +RUN dpkg -i libffi7_3.3-4_amd64.deb +RUN rm libffi7_3.3-4_amd64.deb + +# Make symlinks +RUN ln -s /usr/local/bin/aws /bin/aws +RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh +RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion +RUN /usr/local/bin/cqlsh-expansion.init + +# make binary and script executable +RUN chmod +x /bin/authenticate.sh /bin/delegation-verify + +# set up timezone +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ="Etc/UTC" +RUN apt install tzdata + +# set home to root dir +ENV HOME="/root" + +ENTRYPOINT ["/bin/delegation-verify"] diff --git a/src/app/delegation_verify/README.md b/src/app/delegation_verify/README.md new file mode 100644 index 000000000000..ff147fe9f61c --- /dev/null +++ b/src/app/delegation_verify/README.md @@ -0,0 +1,193 @@ +Stateless delegation verification +--------------------------------- + +This is the stateless verification tool that would be used to verify +that data collected by the uptime service. It verifies the snark work +proofs that can optionally be attached to each submission and also it +validates the blocks claimed to be heads of submitters' best chains at +the time of each submission. + +The tool can work in two distinct modes: +* **file system mode**, where all submission and blocks should be + stored in files on a local file system; +* **cassandra mode**, where all the data is stored in a Cassandra + database. + +Note that submission verification and block validation which this tool +performs closely depends on the version of Mina it was compiled with. +Therefore it is important to always use the verifier tool compiled from +the same revision of the code which produced block and submission one +wants to verify. + +File system mode +---------------- + +Usage: +``` +./delegation-verify fs --block-dir ... +./delegation-verify fs --block-dir - +``` + +Where `` is a file path of the form +`//-.json` +containing JSON following the format described in +[Cloud storage section of delegation backend spec] +(https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage) +(`` is a place holder for some directory): + +```json +{ "created_at": "server's timestamp (of the time of submission)" +, "remote_addr": "ip:port address from which request has come" +, "peer_id": "peer id (as in user's JSON submission)" +, "snark_work": "(optional) base64-encoded snark work blob" +, "block_hash": "base58check-encoded hash of a block" +, "submitter": "base58check-encoded submitter's public key" +} + +``` +File path content: +- `submitted_at_date` with server's date (of the time of submission) in format `YYYY-MM-DD` +- `submitted_at` with server's timestamp (of the time of submission) in RFC-3339 +- `submitter` is base58check-encoded submitter's public key + + +Parameter `--block-dir` specifies the path to the directory containing +block for the submission. It is expected that blocks with +`` exist under path `/.dat` for all +of the filepaths provided. + +When `-` symbol is used as the `filename1`, no other `filename{i}` are +accepted and all filenames will be read from the `stdin`, one filename +per line. + +For each `` parameter one line of output is printed to +`stdout` : + +- Valid payload at `` : + + ```json + { "created_at": "" + , "submitter": "" + , "parent": "" + , "state_hash": "" + , "height": "" + , "slot": "" + } + ``` + +- Invalid payload at `` : + + ```json + { "error": "" } + ``` + +The stateless verification would check the protocol_state_proof in the +block and the transaction_snark_proof in the snark_work. + +Cassandra mode +-------------- + +This mode of operations depends on an external program, `cqlsh` being +installed. It is a Cassandra client written in Python, which +delegation verifier uses to access Cassandra database. This program +should be available in the system path. If it's not, a path to the +`cqlsh` executable can be passed either in `CQLSH` environment +variable or through a command-line option `--clqsh`. Without access +to `cqlsh`, the delegation verifier can only work in the file system +mode described above. + +Additionally, the aforementioned Cassandra client requires some +configuration. In particular, an address and credentials to access +the right Cassandra server must be provided. They should be put in a +configuration file: `$HOME/.cassandra/cqlshrc`. The file can look +like this: + +``` +[authentication] +username = bpu-integration-test-at-673156464838 +password = ************************************ + +[connection] +hostname = cassandra.us-west-2.amazonaws.com +port = 9142 +ssl = true + +[ssl] +certfile = /home/user/uptime-service-backend/database/cert/sf-class2-root.crt +``` + +Alternatively this configuration can be provided in environment variables: +* `CASSANDRA_HOST` +* `CASSANDRA_PORT` +* `CASSANDRA_USERNAME` +* `CASSANDRA_PASSWORD` +* `SSL_CERTFILE` +* `CASSANDRA_USE_SSL` – "1", "YES", "yes", "TRUE", "true" all mean yes, + any other value means no. + +Usage: +``` +./delegation-verify cassandra --keyspace +``` + +Where: +* `keyspace` is the name of the Cassandra keyspace in the AWS cloud +* `start-timestamp` and `end-timestamp` define a time interval from which + submissions will be taken into account. They should be passed in the + format of the `submitted_at` field above (and in the database), that is: + `YYYY-MM-DD HH:mm:ss.0+0000`, where the trailing zeros describe the + timezone, which should be UTC. + +Given this command, the verifier will download submissions from the +given period one by one and verify them. Blocks will also be +downloaded from Cassandra and validated. For each submission, a JSON +statement will be output as shown in the previous +section. Additionally, also appropriate submission records in +Cassandra will be updated with the data. Any errors will also be +logged in Cassandra and **will not** cause the verifier to exit +with a non-zero exit code. + +Fallback to AWS S3 +------------------ + +The Cassandra database (in Amazon Keyspaces) has a limitation +of 1MB per row. Consequently, blocks larger than this size will not be +present in the database. To address this, a fallback mechanism is implemented +to retrieve blocks from AWS S3 when they are not found in Cassandra. +This mode relies on `aws` cli tool that needs to be installed on the system. + +For this mechanism to function properly, the following environment variables must be set: + +* `AWS_CLI` - The path to the AWS CLI tool. If not set, the system defaults to using "/bin/aws". +* `AWS_ACCESS_KEY_ID` - AWS account's access key ID, required for AWS CLI tool to authenticate. +* `AWS_SECRET_ACCESS_KEY` - The secret key paired with access key ID. +* `AWS_S3_BUCKET` - The S3 bucket where submissions and blocks are stored. +* `NETWORK_NAME` - The network name identifier. + +The path to access appropriate block data in S3 is constructed based on these variables as follows: + +`AWS_S3_BUCKET`/`NETWORK_NAME`/blocks/.dat + +Global options +-------------- + +Some command-line options work with both modes of operation: + +* `--no-check` – given this option, the program will skip validation + of blocks and snark work, and will immediately proceed to outputting + state hashes and other metadata for each submission, whether it is + valid or not. + +* `--config-file` - a configuration file of the Mina network. It is + necessary to provide it if the network which produced the blocks + used configuration affecting block validation. In most cases it can + be omitted. + +Build +----- + +To build the docker image go to the root of the repositoryand run + +`docker build -t -f dockerfiles/Dockerfile-delegation-stateless-verifier .` + +where `` is your desired image name. diff --git a/src/app/delegation_verify/cassandra.ml b/src/app/delegation_verify/cassandra.ml new file mode 100644 index 000000000000..bc4296e881e1 --- /dev/null +++ b/src/app/delegation_verify/cassandra.ml @@ -0,0 +1,88 @@ +open Async +open Core + +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or + +type connection_conf = { hostname : string; port : int; use_ssl : bool } + +type credentials = { username : string; password : string } + +type conf = + { executable : string + ; connection : connection_conf option + ; credentials : credentials option + ; keyspace : string + } + +let make_conn_conf () : connection_conf option = + let open Option.Let_syntax in + let%bind hostname = Sys.getenv "CASSANDRA_HOST" in + let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in + let%map use_ssl = + Sys.getenv "CASSANDRA_USE_SSL" + |> Option.map + ~f:(List.mem ~equal:String.equal [ "1"; "TRUE"; "true"; "YES"; "yes" ]) + in + { hostname; port; use_ssl } + +let make_cred_conf () : credentials option = + let open Option.Let_syntax in + let%bind username = Sys.getenv "CASSANDRA_USERNAME" in + let%map password = Sys.getenv "CASSANDRA_PASSWORD" in + { username; password } + +let make_conf ?executable ~keyspace : conf = + let conn = make_conn_conf () in + let credentials = make_cred_conf () in + let executable = + Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const + |> Option.value ~default:"cqlsh" + in + { executable; connection = conn; credentials; keyspace } + +let query ~conf q = + let optional ~f = Option.value_map ~f ~default:[] in + let args = + optional conf.credentials ~f:(fun { username; password } -> + [ "--username"; username; "--password"; password ] ) + @ optional conf.connection ~f:(fun { hostname; port; use_ssl } -> + (if use_ssl then [ "--ssl" ] else []) + @ [ hostname; Int.to_string port ] ) + in + Process.run_lines ~prog:conf.executable ~stdin:q ~args () + +let select ~conf ~parse ~fields ?where from = + let open Deferred.Or_error.Let_syntax in + let%bind data = + query ~conf + @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;" + (String.concat ~sep:"," fields) + conf.keyspace from + (match where with None -> "" | Some w -> " WHERE " ^ w) + in + List.slice data 3 (-2) (* skip header and footer *) + |> List.filter ~f:(fun s -> not (String.is_empty s)) + |> List.fold_right ~init:(Ok []) ~f:(fun line acc -> + let open Or_error.Let_syntax in + let%bind l = acc in + try + let j = Yojson.Safe.from_string line in + match parse j with + | Ppx_deriving_yojson_runtime.Result.Ok s -> + Ok (s :: l) + | Ppx_deriving_yojson_runtime.Result.Error e -> + Or_error.error_string e + with Yojson.Json_error e -> Or_error.error_string e ) + |> Deferred.return + +let update ~conf ~table ~where updates = + let open Deferred.Or_error.Let_syntax in + let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in + let%map _ = + query ~conf + @@ Printf.sprintf "CONSISTENCY LOCAL_QUORUM; UPDATE %s.%s SET %s WHERE %s;" + conf.keyspace table + (String.concat ~sep:"," assignments) + where + in + () diff --git a/src/app/delegation_verify/cassandra.mli b/src/app/delegation_verify/cassandra.mli new file mode 100644 index 000000000000..d2f6247352f1 --- /dev/null +++ b/src/app/delegation_verify/cassandra.mli @@ -0,0 +1,22 @@ +open Async + +type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or + +type conf + +val make_conf : ?executable:string -> keyspace:string -> conf + +val select : + conf:conf + -> parse:'a parser + -> fields:string list + -> ?where:string + -> string + -> 'a list Deferred.Or_error.t + +val update : + conf:conf + -> table:string + -> where:string + -> (string * string) list + -> unit Deferred.Or_error.t diff --git a/src/app/delegation_verify/delegation_verify.ml b/src/app/delegation_verify/delegation_verify.ml new file mode 100644 index 000000000000..6549979b5d12 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.ml @@ -0,0 +1,227 @@ +open Mina_base +open Core +open Async + +let get_filenames = + let open In_channel in + function + | [ "-" ] | [] -> + input_all stdin |> String.split_lines + | filenames -> + filenames + +let verify_snark_work ~verify_transaction_snarks ~proof ~message = + verify_transaction_snarks [ (proof, message) ] + +let config_flag = + let open Command.Param in + flag "--config-file" ~doc:"FILE config file" (optional string) + +let keyspace_flag = + let open Command.Param in + flag "--keyspace" ~doc:"Name of the Cassandra keyspace" (required string) + +let no_checks_flag = + let open Command.Param in + flag "--no-checks" ~aliases:[ "-no-checks" ] + ~doc:"disable all the checks, just extract the info from the submissions" + no_arg + +let block_dir_flag = + let open Command.Param in + flag "--block-dir" ~aliases:[ "-block-dir" ] + ~doc:"the path to the directory containing blocks for the submission" + (required Filename.arg_type) + +let cassandra_executable_flag = + let open Command.Param in + flag "--executable" + ~aliases:[ "-executable"; "--cqlsh"; "-cqlsh" ] + ~doc:"the path to the cqlsh executable" + (optional Filename.arg_type) + +let timestamp = + let open Command.Param in + anon ("timestamp" %: string) + +let instantiate_verify_functions ~logger = function + | None -> + Deferred.return + (Verifier.verify_functions + ~constraint_constants:Genesis_constants.Constraint_constants.compiled + ~proof_level:Genesis_constants.Proof_level.compiled () ) + | Some config_file -> + let%bind.Deferred precomputed_values = + let%bind.Deferred.Or_error config_json = + Genesis_ledger_helper.load_config_json config_file + in + let%bind.Deferred.Or_error config = + Deferred.return + @@ Result.map_error ~f:Error.of_string + @@ Runtime_config.of_yojson config_json + in + Genesis_ledger_helper.init_from_config_file ~logger ~proof_level:None + config + in + let%map.Deferred precomputed_values = + match precomputed_values with + | Ok (precomputed_values, _) -> + Deferred.return precomputed_values + | Error _ -> + Output.display_error "fail to read config file" ; + exit 4 + in + let constraint_constants = + Precomputed_values.constraint_constants precomputed_values + in + Verifier.verify_functions ~constraint_constants ~proof_level:Full () + +module Make_verifier (Source : Submission.Data_source) = struct + let verify_transaction_snarks = Source.verify_transaction_snarks + + let verify_blockchain_snarks = Source.verify_blockchain_snarks + + let intialize_submission ?validate (src : Source.t) (sub : Source.submission) + = + let block_hash = Source.block_hash sub in + if Known_blocks.is_known block_hash then () + else + let load_block_action = + if Source.is_cassandra src then Source.load_block_from_submission sub + else Source.load_block src ~block_hash + in + Known_blocks.add ?validate ~verify_blockchain_snarks ~block_hash + load_block_action + + let verify ~validate (submission : Source.submission) = + let open Deferred.Result.Let_syntax in + let block_hash = Source.block_hash submission in + let%bind block = Known_blocks.get block_hash in + let%bind () = Known_blocks.is_valid block_hash in + let%map () = + if validate then + match Source.snark_work submission with + | None -> + Deferred.Result.return () + | Some + Uptime_service.Proof_data.{ proof; proof_time = _; snark_work_fee } + -> + let message = + Mina_base.Sok_message.create ~fee:snark_work_fee + ~prover:(Source.submitter submission) + in + verify_snark_work ~verify_transaction_snarks ~proof ~message + else return () + in + let header = Mina_block.header block in + let protocol_state = Mina_block.Header.protocol_state header in + let consensus_state = + Mina_state.Protocol_state.consensus_state protocol_state + in + ( Mina_state.Protocol_state.hashes protocol_state + |> State_hash.State_hashes.state_hash + , Mina_state.Protocol_state.previous_state_hash protocol_state + , Consensus.Data.Consensus_state.blockchain_length consensus_state + , Consensus.Data.Consensus_state.global_slot_since_genesis consensus_state + ) + + let validate_and_display_results ~validate ~src submission = + let open Deferred.Let_syntax in + let%bind result = verify ~validate submission in + Result.map result ~f:(fun (state_hash, parent, height, slot) -> + Output. + { submitted_at = Source.submitted_at submission + ; submitter = + Signature_lib.Public_key.Compressed.to_base58_check + (Source.submitter submission) + ; state_hash + ; parent + ; height + ; slot + } ) + |> Source.output src submission + + let process ?(validate = true) (src : Source.t) = + let open Deferred.Or_error.Let_syntax in + let%bind submissions = Source.load_submissions src in + List.iter submissions ~f:(intialize_submission ~validate src) ; + List.map submissions ~f:(validate_and_display_results ~src ~validate) + |> Deferred.Or_error.all_unit +end + +let filesystem_command = + Command.async ~summary:"Verify submissions and block read from the filesystem" + Command.Let_syntax.( + let%map_open block_dir = block_dir_flag + and inputs = anon (sequence ("filename" %: Filename.arg_type)) + and no_checks = no_checks_flag + and config_file = config_flag in + fun () -> + let logger = Logger.create () in + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = + instantiate_verify_functions ~logger config_file + in + let submission_paths = get_filenames inputs in + let module V = Make_verifier (struct + include Submission.Filesystem + + let is_cassandra _ = false + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in + let open Deferred.Let_syntax in + match%bind + V.process ~validate:(not no_checks) { submission_paths; block_dir } + with + | Ok () -> + Deferred.unit + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + exit 1) + +let cassandra_command = + Command.async ~summary:"Verify submissions and block read from Cassandra" + Command.Let_syntax.( + let%map_open cqlsh = cassandra_executable_flag + and no_checks = no_checks_flag + and config_file = config_flag + and keyspace = keyspace_flag + and period_start = timestamp + and period_end = timestamp in + fun () -> + let open Deferred.Let_syntax in + let logger = Logger.create () in + let%bind.Deferred verify_blockchain_snarks, verify_transaction_snarks = + instantiate_verify_functions ~logger config_file + in + let module V = Make_verifier (struct + include Submission.Cassandra + + let is_cassandra _ = true + + let verify_blockchain_snarks = verify_blockchain_snarks + + let verify_transaction_snarks = verify_transaction_snarks + end) in + let src = + Submission.Cassandra. + { conf = Cassandra.make_conf ?executable:cqlsh ~keyspace + ; period_start + ; period_end + } + in + match%bind V.process ~validate:(not no_checks) src with + | Ok () -> + Deferred.unit + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + exit 1) + +let command = + Command.group + ~summary:"A tool for verifying JSON payload submitted by the uptime service" + [ ("fs", filesystem_command); ("cassandra", cassandra_command) ] + +let () = Async.Command.run command diff --git a/src/app/delegation_verify/delegation_verify.opam b/src/app/delegation_verify/delegation_verify.opam new file mode 100644 index 000000000000..7be19e3d6129 --- /dev/null +++ b/src/app/delegation_verify/delegation_verify.opam @@ -0,0 +1,5 @@ +opam-version: "2.0" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune new file mode 100644 index 000000000000..3de6f7938743 --- /dev/null +++ b/src/app/delegation_verify/dune @@ -0,0 +1,49 @@ +(executable + (name delegation_verify) + (libraries + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + sexplib0 + sexplib + hex + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base_import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_backend.pasta + kimchi_backend.pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger + ) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/delegation_verify/known_blocks.ml b/src/app/delegation_verify/known_blocks.ml new file mode 100644 index 000000000000..92913e964167 --- /dev/null +++ b/src/app/delegation_verify/known_blocks.ml @@ -0,0 +1,67 @@ +open Async +open Core + +module Block_hash = struct + open Sexplib.Conv + + type t = string [@@deriving sexp, compare] + + let hash = Base.String.hash +end + +module Deferred_block = struct + type t = + { block : Mina_block.t Deferred.Or_error.t + ; valid : unit Deferred.Or_error.t (* Raises if block is invalid. *) + } + + let block_of_string contents = + let compute v = + let r = + try Ok (Binable.of_string (module Mina_block.Stable.Latest) contents) + with _ -> Error (Error.of_string "Fail to decode block") + in + Async.Ivar.fill v r + in + Deferred.create compute + + let verify_block ~verify_blockchain_snarks block = + let header = Mina_block.header block in + let open Mina_block.Header in + verify_blockchain_snarks + [ (protocol_state header, protocol_state_proof header) ] + + let create ?(validate = true) ~verify_blockchain_snarks contents = + let open Deferred.Or_error.Monad_infix in + let block = contents >>= block_of_string in + let valid = + if validate then block >>= verify_block ~verify_blockchain_snarks + else Deferred.Or_error.return () + in + { block; valid } +end + +let known_blocks : (Block_hash.t, Deferred_block.t) Hashtbl.t = + Hashtbl.create (module Block_hash) + +let add ?validate ~verify_blockchain_snarks ~block_hash block_contents = + let block = + Deferred_block.create ?validate ~verify_blockchain_snarks block_contents + in + Hashtbl.add_exn known_blocks ~key:block_hash ~data:block + +let is_known hash = Hashtbl.mem known_blocks hash + +let get hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.block + +let is_valid hash = + match Hashtbl.find known_blocks hash with + | None -> + Deferred.Or_error.errorf "Block %s not found" hash + | Some b -> + b.valid diff --git a/src/app/delegation_verify/output.ml b/src/app/delegation_verify/output.ml new file mode 100644 index 000000000000..74a763110c74 --- /dev/null +++ b/src/app/delegation_verify/output.ml @@ -0,0 +1,39 @@ +open Async +open Mina_base + +type t = + { submitted_at : string + ; submitter : string + ; state_hash : State_hash.t + ; parent : State_hash.t + ; height : Unsigned.uint32 + ; slot : Mina_numbers.Global_slot_since_genesis.t + } + +let valid_payload_to_yojson (p : t) : Yojson.Safe.t = + `Assoc + [ ("submitted_at", `String p.submitted_at) + ; ("submitter", `String p.submitter) + ; ("state_hash", State_hash.to_yojson p.state_hash) + ; ("parent", State_hash.to_yojson p.parent) + ; ("height", `Int (Unsigned.UInt32.to_int p.height)) + ; ("slot", `Int (Mina_numbers.Global_slot_since_genesis.to_int p.slot)) + ] + +let valid_payload_to_cassandra_updates (p : t) = + [ ("height", Unsigned.UInt32.to_string p.height) + ; ("slot", Mina_numbers.Global_slot_since_genesis.to_string p.slot) + ; ("parent", Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.parent) + ; ( "state_hash" + , Printf.sprintf "'%s'" @@ State_hash.to_base58_check p.state_hash ) + ; ("raw_block", "NULL") + ; ("snark_work", "NULL") + ; ("verified", "true") + ] + +let display valid_payload = + printf "%s\n" @@ Yojson.Safe.to_string + @@ valid_payload_to_yojson valid_payload + +let display_error e = + eprintf "%s\n" @@ Yojson.Safe.to_string @@ `Assoc [ ("error", `String e) ] diff --git a/src/app/delegation_verify/scripts/authenticate.sh b/src/app/delegation_verify/scripts/authenticate.sh new file mode 100644 index 000000000000..331a84fdafbf --- /dev/null +++ b/src/app/delegation_verify/scripts/authenticate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -x + +# The environment variables are set at the time of deployment in the init container +CREDENTIALS=$(aws sts assume-role --role-session-name $AWS_ROLE_SESSION_NAME --role-arn $AWS_ROLE_ARN) + +ACCESS_KEY_ID=$(echo $CREDENTIALS | jq -r '.Credentials.AccessKeyId') + +SECRET_ACCESS_KEY=$(echo $CREDENTIALS | jq -r '.Credentials.SecretAccessKey') + +SESSION_TOKEN=$(echo $CREDENTIALS | jq -r '.Credentials.SessionToken') + +echo "export AWS_ACCESS_KEY_ID=$ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$SECRET_ACCESS_KEY AWS_SESSION_TOKEN=$SESSION_TOKEN" > /var/mina-delegation-verify-auth/.env diff --git a/src/app/delegation_verify/submission.ml b/src/app/delegation_verify/submission.ml new file mode 100644 index 000000000000..80af3f12af4f --- /dev/null +++ b/src/app/delegation_verify/submission.ml @@ -0,0 +1,318 @@ +open Async +open Core +open Signature_lib + +let decode_snark_work str = + match Base64.decode str with + | Ok str -> ( + try Ok (Binable.of_string (module Uptime_service.Proof_data) str) + with _ -> Error (Error.of_string "Fail to decode snark work") ) + | Error _ -> + Error (Error.of_string "Fail to decode snark work") + +module type Data_source = sig + type t + + type submission + + val is_cassandra : t -> bool + + val submitted_at : submission -> string + + val block_hash : submission -> string + + val snark_work : submission -> Uptime_service.Proof_data.t option + + val submitter : submission -> Public_key.Compressed.t + + val load_submissions : t -> submission list Deferred.Or_error.t + + val load_block : block_hash:string -> t -> string Deferred.Or_error.t + + val load_block_from_submission : submission -> string Deferred.Or_error.t + + val verify_blockchain_snarks : + (Mina_wire_types.Mina_state_protocol_state.Value.V2.t * Mina_base.Proof.t) + list + -> unit Async_kernel__Deferred_or_error.t + + val verify_transaction_snarks : + (Ledger_proof.t * Mina_base.Sok_message.t) list + -> (unit, Error.t) Deferred.Result.t + + val output : + t -> submission -> Output.t Or_error.t -> unit Deferred.Or_error.t +end + +module Filesystem = struct + type t = { block_dir : string; submission_paths : string list } + + type submission = + { submitted_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + { submitter + ; snark_work + ; block_hash = meta.block_hash + ; submitted_at = meta.created_at + } + + let submitted_at ({ submitted_at; _ } : submission) = submitted_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + + let load_submissions { submission_paths; _ } = + Deferred.create (fun ivar -> + List.fold_right submission_paths ~init:(Ok []) ~f:(fun filename acc -> + let open Result.Let_syntax in + let%bind acc = acc in + let%bind contents = + try Ok (In_channel.read_all filename) + with _ -> Error (Error.of_string "Fail to load metadata") + in + let%bind meta = + match Yojson.Safe.from_string contents |> raw_of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok a -> + Ok a + | Ppx_deriving_yojson_runtime.Result.Error e -> + Error (Error.of_string e) + in + let%map t = of_raw meta in + t :: acc ) + |> Ivar.fill ivar ) + + let load_block ~block_hash { block_dir; _ } = + Deferred.create (fun ivar -> + let block_path = Printf.sprintf "%s/%s.dat" block_dir block_hash in + ( try Ok (In_channel.read_all block_path) + with _ -> Error (Error.of_string "Fail to load block") ) + |> Ivar.fill ivar ) + + (* Dummy impl, not to be used in the context of Filesystem module. + It is only intended to fulfill the interface requirements of the + Data_source module signature for the Filesystem module *) + let load_block_from_submission submission = + let _ = submission in + Deferred.Or_error.return "dummy block data" + + let output _ (_submission : submission) = function + | Ok payload -> + Output.display payload ; + Deferred.Or_error.return () + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Deferred.Or_error.return () +end + +module Cassandra = struct + type t = { conf : Cassandra.conf; period_start : string; period_end : string } + + type block_data = { raw_block : string } [@@deriving yojson] + + type submission = + { created_at : string + ; snark_work : Uptime_service.Proof_data.t option + ; submitter : Public_key.Compressed.t + ; block_hash : string + ; submitted_at : string + ; submitted_at_date : string + ; raw_block : string option [@default None] + } + + type raw = + { created_at : string + ; peer_id : string + ; snark_work : string option [@default None] + ; remote_addr : string + ; submitter : string + ; block_hash : string + ; graphql_control_port : int option [@default None] + ; submitted_at : string + ; submitted_at_date : string + ; raw_block : string option [@default None] + } + [@@deriving yojson] + + let of_raw meta = + let open Result.Let_syntax in + let%bind.Result submitter = + Public_key.Compressed.of_base58_check meta.submitter + in + let%map snark_work = + match meta.snark_work with + | None -> + Ok None + | Some s -> + let%map snark_work = decode_snark_work s in + Some snark_work + in + ( { submitter + ; snark_work + ; block_hash = meta.block_hash + ; created_at = meta.created_at + ; submitted_at_date = meta.submitted_at_date + ; submitted_at = meta.submitted_at + ; raw_block = meta.raw_block + } + : submission ) + + let submitted_at ({ submitted_at; _ } : submission) = submitted_at + + let block_hash ({ block_hash; _ } : submission) = block_hash + + let snark_work ({ snark_work; _ } : submission) = snark_work + + let submitter ({ submitter; _ } : submission) = submitter + + let load_submissions { conf; period_start; period_end } = + let open Deferred.Or_error.Let_syntax in + let start_day = + Time.of_string period_start |> Time.to_date ~zone:Time.Zone.utc + in + let end_day = + Time.of_string period_end |> Time.to_date ~zone:Time.Zone.utc + in + let partition_keys = + Date.dates_between ~min:start_day ~max:end_day + |> List.map ~f:(fun d -> Date.format d "%Y-%m-%d") + in + let partition = + if List.length partition_keys = 1 then + sprintf "submitted_at_date = '%s'" (List.hd_exn partition_keys) + else + sprintf "submitted_at_date IN (%s)" + (String.concat ~sep:"," @@ List.map ~f:(sprintf "'%s'") partition_keys) + in + let%bind raw = + Cassandra.select ~conf ~parse:raw_of_yojson + ~fields: + [ "created_at" + ; "submitted_at_date" + ; "submitted_at" + ; "peer_id" + ; "snark_work" + ; "remote_addr" + ; "submitter" + ; "block_hash" + ; "graphql_control_port" + ; "raw_block" + ] + ~where: + (sprintf "%s AND submitted_at >= '%s' AND submitted_at < '%s'" + partition period_start period_end ) + "submissions" + in + List.fold_right raw ~init:(Ok []) ~f:(fun sub acc -> + let open Result.Let_syntax in + let%bind l = acc in + let snark_work = + Option.map sub.snark_work ~f:(fun s -> + String.chop_prefix_exn s ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> Base64.encode_string ) + in + let%map s = of_raw { sub with snark_work } in + s :: l ) + |> Deferred.return + + let load_from_s3 ~block_hash = + let aws_cli = Option.value ~default:"/bin/aws" @@ Sys.getenv "AWS_CLI" in + let s3_path = + let open Or_error.Let_syntax in + let%bind bucket = + Or_error.try_with (fun () -> Sys.getenv_exn "AWS_S3_BUCKET") + in + let%map network = + Or_error.try_with (fun () -> Sys.getenv_exn "NETWORK_NAME") + in + sprintf "s3://%s/%s/blocks/%s.dat" bucket network block_hash + in + Deferred.Or_error.bind (return s3_path) ~f:(fun s3_path -> + Process.run ~prog:aws_cli ~args:[ "s3"; "cp"; s3_path; "-" ] () ) + + let load_block_from_submission (submission : submission) = + let open Deferred.Or_error.Let_syntax in + match submission.raw_block with + | None -> + (* If not found in Submission, try loading from S3 *) + load_from_s3 ~block_hash:submission.block_hash + | Some b -> + String.chop_prefix_exn b ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> return + + (* The 'blocks' table is no longer actively used in the Cassandra schema. + However, 'load_block' is retained for reference purposes and in case of + schema rollbacks or data migration needs. *) + let load_block ~block_hash { conf; _ } = + let open Deferred.Or_error.Let_syntax in + let%bind block_data = + Cassandra.select ~conf ~parse:block_data_of_yojson ~fields:[ "raw_block" ] + ~where:(sprintf "block_hash = '%s'" block_hash) + "blocks" + in + match List.hd block_data with + | None -> + (* If not found in Cassandra, try loading from S3 *) + load_from_s3 ~block_hash + | Some b -> + String.chop_prefix_exn b.raw_block ~prefix:"0x" + |> Hex.Safe.of_hex |> Option.value_exn |> return + + let output { conf; _ } (submission : submission) = function + | Ok payload -> + Output.display payload ; + Cassandra.update ~conf ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) + submission.submitted_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + Output.(valid_payload_to_cassandra_updates payload) + | Error e -> + Output.display_error @@ Error.to_string_hum e ; + Cassandra.update ~conf ~table:"submissions" + ~where: + (sprintf + "submitted_at_date = '%s' and submitted_at = '%s' and submitter \ + = '%s'" + (List.hd_exn @@ String.split ~on:' ' submission.submitted_at) + submission.submitted_at + (Public_key.Compressed.to_base58_check submission.submitter) ) + [ ("validation_error", sprintf "'%s'" (Error.to_string_hum e)) + ; ("raw_block", "NULL") + ; ("snark_work", "NULL") + ; ("verified", "true") + ] +end diff --git a/src/app/delegation_verify/verifier.ml b/src/app/delegation_verify/verifier.ml new file mode 100644 index 000000000000..43677f5d6405 --- /dev/null +++ b/src/app/delegation_verify/verifier.ml @@ -0,0 +1,14 @@ +let verify_functions ~constraint_constants ~proof_level () = + let module T = Transaction_snark.Make (struct + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + (B.Proof.verify, T.verify)