The stateless verification tool

dockerfiles/Dockerfile-delegation-stateless-verifier

@@ -0,0 +1,105 @@
+#################################################################################################
+# The "stateless verification build" Stage
+# - builds stateless verification tool
+# - should not include any data related to joining a specific network, only the node software itself
+#################################################################################################
+FROM gcr.io/o1labs-192920/mina-toolchain@sha256:c810338e2c3973f7c674d0607048725917ce2be23b949c4bc9760c01122f884b AS builder
+
+# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI
+ARG DUNE_PROFILE=devnet
+
+# branch to checkout on first clone (this will be the only availible branch in the container)
+# can also be a tagged release
+ARG MINA_BRANCH=sventimir/stateless-verification-tool
+
+# repo to checkout the branch from
+ARG MINA_REPO=https://github.com/MinaProtocol/mina
+
+# location of repo used for pins and external package commits
+ARG MINA_DIR=mina
+
+ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin"
+
+# git will clone into an empty dir, but this also helps us set the workdir in advance
+RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \
+  && git clone \
+  -b "${MINA_BRANCH}" \
+  --depth 1 \
+  --shallow-submodules \
+  --recurse-submodules \
+  ${MINA_REPO} ${HOME}/${MINA_DIR}
+
+WORKDIR $HOME/${MINA_DIR}
+
+RUN git submodule sync && git submodule update --init --recursive
+
+RUN mkdir ${HOME}/app
+
+# HACK: build without special cpu features to allow more people to run delegation verification tool
+# RUN ./scripts/zexe-standardize.sh
+
+RUN eval $(opam config env) \
+  && dune build --profile=${DUNE_PROFILE} \
+    src/app/delegation_verify/delegation_verify.exe \
+  && cp _build/default/src/app/delegation_verify/delegation_verify.exe ./delegation-verify \
+  && rm -rf _build
+
+USER root
+
+# copy binary to /bin
+RUN cp ./delegation-verify /bin/delegation-verify
+
+# add authenticate.sh to image
+RUN cp src/app/delegation_verify/scripts/authenticate.sh /bin/authenticate.sh
+
+# Runtime image
+FROM ubuntu:latest
+
+# Copy resources from builder to runtime image
+COPY --from=builder /bin/delegation-verify /bin/delegation-verify
+COPY --from=builder /bin/authenticate.sh /bin/authenticate.sh
+
+# awscli and cqlsh-expansion are used by the delegation verification tool
+RUN apt-get update && apt-get install -y python3 python3-pip jq libjemalloc2 wget dnsutils gawk
+RUN pip3 install awscli
+RUN pip3 install cqlsh-expansion
+RUN pip3 install pytz
+
+# Install libssl1.1.1b (not in apt)
+RUN wget https://www.openssl.org/source/openssl-1.1.1b.tar.gz
+RUN mkdir /opt/openssl
+RUN tar xfvz openssl-1.1.1b.tar.gz --directory /opt/openssl
+RUN rm openssl-1.1.1b.tar.gz
+
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/openssl/lib"
+ENV PATH="$PATH:/opt/openssl/bin"
+
+RUN cd /opt/openssl/openssl-1.1.1b && ./config --prefix=/opt/openssl --openssldir=/opt/openssl/ssl
+RUN cd /opt/openssl/openssl-1.1.1b && make && make install
+
+# Rename openssl old binary
+RUN mv /usr/bin/openssl /usr/bin/openssl.old
+
+# Install libffi7
+RUN wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb
+RUN dpkg -i libffi7_3.3-4_amd64.deb
+RUN rm libffi7_3.3-4_amd64.deb
+
+# Make symlinks
+RUN ln -s /usr/local/bin/aws /bin/aws
+RUN ln -s /usr/local/bin/cqlsh /bin/cqlsh
+RUN ln -s /usr/local/bin/cqlsh-expansion /bin/cqlsh-expansion
+RUN /usr/local/bin/cqlsh-expansion.init
+
+# make binary and script executable
+RUN chmod +x /bin/authenticate.sh /bin/delegation-verify
+
+# set up timezone
+ENV DEBIAN_FRONTEND="noninteractive"
+ENV TZ="Etc/UTC"
+RUN apt install tzdata
+
+# set home to root dir
+ENV HOME="/root"
+
+ENTRYPOINT ["/bin/delegation-verify"]

src/app/delegation_verify/README.md

@@ -0,0 +1,193 @@
+Stateless delegation verification
+---------------------------------
+
+This is the stateless verification tool that would be used to verify
+that data collected by the uptime service. It verifies the snark work
+proofs that can optionally be attached to each submission and also it
+validates the blocks claimed to be heads of submitters' best chains at
+the time of each submission.
+
+The tool can work in two distinct modes:
+* **file system mode**, where all submission and blocks should be
+  stored in files on a local file system;
+* **cassandra mode**, where all the data is stored in a Cassandra
+  database.
+
+Note that submission verification and block validation which this tool
+performs closely depends on the version of Mina it was compiled with.
+Therefore it is important to always use the verifier tool compiled from
+the same revision of the code which produced block and submission one
+wants to verify.
+
+File system mode
+----------------
+
+Usage:
+```
+./delegation-verify fs --block-dir <blockdir> <filepath1> <filepath2> ... <filepath{n}>
+./delegation-verify fs --block-dir <blockdir> -
+```
+
+Where `<filepath{i}>` is a file path of the form
+`<basedir>/<submitted_at_date>/<submitted_at>-<submitter>.json`
+containing JSON following the format described in
+[Cloud storage section of delegation backend spec]
+(https://github.com/MinaProtocol/mina/blob/develop/src/app/delegation_backend/README.md#cloud-storage)
+(`<basedir>` is a place holder for some directory):
+
+```json
+{ "created_at": "server's timestamp (of the time of submission)"
+, "remote_addr": "ip:port address from which request has come"
+, "peer_id": "peer id (as in user's JSON submission)"
+, "snark_work": "(optional) base64-encoded snark work blob"
+, "block_hash": "base58check-encoded hash of a block"
+, "submitter": "base58check-encoded submitter's public key"
+}
+
+```
+File path content:
+- `submitted_at_date` with server's date (of the time of submission) in format `YYYY-MM-DD`
+- `submitted_at` with server's timestamp (of the time of submission) in RFC-3339
+- `submitter` is base58check-encoded submitter's public key
+
+
+Parameter `--block-dir` specifies the path to the directory containing
+block for the submission. It is expected that blocks with
+`<block_hash>` exist under path `<blockdir>/<block_hash>.dat` for all
+of the filepaths provided.
+
+When `-` symbol is used as the `filename1`, no other `filename{i}` are
+accepted and all filenames will be read from the `stdin`, one filename
+per line.
+
+For each `<filename{i}>` parameter one line of output is printed to
+`stdout` :
+
+- Valid payload at `<filename{i}>` :
+
+    ```json
+    { "created_at": "<timestamp of the submission (for identification)>"
+    , "submitter": "<submitter's public key (for identification)>"
+    , "parent": "<hash of the parent block>"
+    , "state_hash": "<state hash of the block in base58 format>"
+    , "height": "<blockchain height>"
+    , "slot": "<global slot since genesis corresponding to the block>"
+    }
+    ```
+
+- Invalid payload at `<filename{i}>` :
+
+    ```json
+    { "error": "<reason of rejection>" }
+    ```
+
+The stateless verification would check the protocol_state_proof in the
+block and the transaction_snark_proof in the snark_work.
+
+Cassandra mode
+--------------
+
+This mode of operations depends on an external program, `cqlsh` being
+installed. It is a Cassandra client written in Python, which
+delegation verifier uses to access Cassandra database. This program
+should be available in the system path. If it's not, a path to the
+`cqlsh` executable can be passed either in `CQLSH` environment
+variable or through a command-line option `--clqsh`. Without access
+to `cqlsh`, the delegation verifier can only work in the file system
+mode described above.
+
+Additionally, the aforementioned Cassandra client requires some
+configuration. In particular, an address and credentials to access
+the right Cassandra server must be provided. They should be put in a
+configuration file: `$HOME/.cassandra/cqlshrc`. The file can look
+like this:
+
+```
+[authentication]
+username = bpu-integration-test-at-673156464838
+password = ************************************
+
+[connection]
+hostname = cassandra.us-west-2.amazonaws.com
+port = 9142
+ssl = true
+
+[ssl]
+certfile = /home/user/uptime-service-backend/database/cert/sf-class2-root.crt
+```
+
+Alternatively this configuration can be provided in environment variables:
+* `CASSANDRA_HOST`
+* `CASSANDRA_PORT`
+* `CASSANDRA_USERNAME`
+* `CASSANDRA_PASSWORD`
+* `SSL_CERTFILE`
+* `CASSANDRA_USE_SSL` – "1", "YES", "yes", "TRUE", "true" all mean yes,
+  any other value means no.
+
+Usage:
+```
+./delegation-verify cassandra --keyspace <keyspace-name> <start-timestamp> <end-timstamp>
+```
+
+Where:
+* `keyspace` is the name of the Cassandra keyspace in the AWS cloud
+* `start-timestamp` and `end-timestamp` define a time interval from which
+  submissions will be taken into account. They should be passed in the
+  format of the `submitted_at` field above (and in the database), that is:
+  `YYYY-MM-DD HH:mm:ss.0+0000`, where the trailing zeros describe the
+  timezone, which should be UTC.
+
+Given this command, the verifier will download submissions from the
+given period one by one and verify them. Blocks will also be
+downloaded from Cassandra and validated. For each submission, a JSON
+statement will be output as shown in the previous
+section. Additionally, also appropriate submission records in
+Cassandra will be updated with the data. Any errors will also be
+logged in Cassandra and **will not** cause the verifier to exit
+with a non-zero exit code.
+
+Fallback to AWS S3
+------------------
+
+The Cassandra database (in Amazon Keyspaces) has a limitation 
+of 1MB per row. Consequently, blocks larger than this size will not be 
+present in the database. To address this, a fallback mechanism is implemented 
+to retrieve blocks from AWS S3 when they are not found in Cassandra.
+This mode relies on `aws` cli tool that needs to be installed on the system.
+
+For this mechanism to function properly, the following environment variables must be set:
+
+* `AWS_CLI` - The path to the AWS CLI tool. If not set, the system defaults to using "/bin/aws".
+* `AWS_ACCESS_KEY_ID` - AWS account's access key ID, required for AWS CLI tool to authenticate.
+* `AWS_SECRET_ACCESS_KEY` - The secret key paired with access key ID.
+* `AWS_S3_BUCKET` - The S3 bucket where submissions and blocks are stored.
+* `NETWORK_NAME` - The network name identifier.
+
+The path to access appropriate block data in S3 is constructed based on these variables as follows:
+
+`AWS_S3_BUCKET`/`NETWORK_NAME`/blocks/<block_hash>.dat
+
+Global options
+--------------
+
+Some command-line options work with both modes of operation:
+
+* `--no-check` – given this option, the program will skip validation
+  of blocks and snark work, and will immediately proceed to outputting
+  state hashes and other metadata for each submission, whether it is
+  valid or not.
+
+* `--config-file` - a configuration file of the Mina network. It is
+  necessary to provide it if the network which produced the blocks
+  used configuration affecting block validation. In most cases it can
+  be omitted.
+
+Build
+-----
+
+To build the docker image go to the root of the repositoryand run
+
+`docker build -t <image-name> -f dockerfiles/Dockerfile-delegation-stateless-verifier .`
+
+where `<image-name>` is your desired image name.

src/app/delegation_verify/cassandra.ml

@@ -0,0 +1,88 @@
+open Async
+open Core
+
+type 'a parser = Yojson.Safe.t -> 'a Ppx_deriving_yojson_runtime.error_or
+
+type connection_conf = { hostname : string; port : int; use_ssl : bool }
+
+type credentials = { username : string; password : string }
+
+type conf =
+  { executable : string
+  ; connection : connection_conf option
+  ; credentials : credentials option
+  ; keyspace : string
+  }
+
+let make_conn_conf () : connection_conf option =
+  let open Option.Let_syntax in
+  let%bind hostname = Sys.getenv "CASSANDRA_HOST" in
+  let%bind port = Option.map ~f:Int.of_string @@ Sys.getenv "CASSANDRA_PORT" in
+  let%map use_ssl =
+    Sys.getenv "CASSANDRA_USE_SSL"
+    |> Option.map
+         ~f:(List.mem ~equal:String.equal [ "1"; "TRUE"; "true"; "YES"; "yes" ])
+  in
+  { hostname; port; use_ssl }
+
+let make_cred_conf () : credentials option =
+  let open Option.Let_syntax in
+  let%bind username = Sys.getenv "CASSANDRA_USERNAME" in
+  let%map password = Sys.getenv "CASSANDRA_PASSWORD" in
+  { username; password }
+
+let make_conf ?executable ~keyspace : conf =
+  let conn = make_conn_conf () in
+  let credentials = make_cred_conf () in
+  let executable =
+    Option.merge executable (Sys.getenv "CQLSH") ~f:Fn.const
+    |> Option.value ~default:"cqlsh"
+  in
+  { executable; connection = conn; credentials; keyspace }
+
+let query ~conf q =
+  let optional ~f = Option.value_map ~f ~default:[] in
+  let args =
+    optional conf.credentials ~f:(fun { username; password } ->
+        [ "--username"; username; "--password"; password ] )
+    @ optional conf.connection ~f:(fun { hostname; port; use_ssl } ->
+          (if use_ssl then [ "--ssl" ] else [])
+          @ [ hostname; Int.to_string port ] )
+  in
+  Process.run_lines ~prog:conf.executable ~stdin:q ~args ()
+
+let select ~conf ~parse ~fields ?where from =
+  let open Deferred.Or_error.Let_syntax in
+  let%bind data =
+    query ~conf
+    @@ Printf.sprintf "SELECT JSON %s FROM %s.%s%s;"
+         (String.concat ~sep:"," fields)
+         conf.keyspace from
+         (match where with None -> "" | Some w -> " WHERE " ^ w)
+  in
+  List.slice data 3 (-2) (* skip header and footer *)
+  |> List.filter ~f:(fun s -> not (String.is_empty s))
+  |> List.fold_right ~init:(Ok []) ~f:(fun line acc ->
+         let open Or_error.Let_syntax in
+         let%bind l = acc in
+         try
+           let j = Yojson.Safe.from_string line in
+           match parse j with
+           | Ppx_deriving_yojson_runtime.Result.Ok s ->
+               Ok (s :: l)
+           | Ppx_deriving_yojson_runtime.Result.Error e ->
+               Or_error.error_string e
+         with Yojson.Json_error e -> Or_error.error_string e )
+  |> Deferred.return
+
+let update ~conf ~table ~where updates =
+  let open Deferred.Or_error.Let_syntax in
+  let assignments = List.map updates ~f:(fun (k, v) -> k ^ " = " ^ v) in
+  let%map _ =
+    query ~conf
+    @@ Printf.sprintf "CONSISTENCY LOCAL_QUORUM; UPDATE %s.%s SET %s WHERE %s;"
+         conf.keyspace table
+         (String.concat ~sep:"," assignments)
+         where
+  in
+  ()