fix: add certs to container and check for connections in build

.github/workflows/build-and-test.yml

@@ -46,7 +46,7 @@ jobs:
     - uses: mozilla-actions/[email protected]
     - name: Run tests
       run: |
-        echo "Skipping tests"
+        echo "Skipping tests since github won't let us use tmp directories"
     env:
       DID_DOCUMENT: ${{ vars.DID_DOCUMENT }}
       DID_PRIVATE_KEY: ${{ secrets.DID_PRIVATE_KEY }}
@@ -67,3 +67,18 @@ jobs:
     - uses: mozilla-actions/[email protected]
     - name: Build
       run: make build
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build Image
+        run: |
+          make build-docker
+    env:
+      DID_DOCUMENT: ${{ vars.DID_DOCUMENT }}
+      DID_PRIVATE_KEY: ${{ secrets.DID_PRIVATE_KEY }}
+      RUST_LOG: info
+      CERAMIC_URL: ${{ vars.CERAMIC_URL }}
+

.github/workflows/publish.yml

@@ -30,6 +30,7 @@ jobs:
           password: ${{ env.AWS_SECRET_ACCESS_KEY }}
         env:
           AWS_REGION: us-east-1
+          CERAMIC_URL: ${{ vars.CERAMIC_URL }}
       -
         name: Build and Publish
         id: build

Cargo.toml

@@ -13,6 +13,8 @@ members = [
   anyhow = "1.0.80"
   async-trait = "0.1.77"
   base64 = "0.21.7"
+  clap = { version = "=4.4.18", default-features = false, features = ["derive", "std"] }
+  curve25519-dalek = "=4.1.1"
   hmac = "0.12.1"
   jwt = "0.16.0"
   log = "0.4.20"

Dockerfile

@@ -21,6 +21,8 @@ RUN --mount=type=cache,target=/home/builder/.cargo \
 
 FROM --platform=linux/amd64 debian:bookworm-slim
 
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+
 COPY --from=builder /home/builder/checkpointer/bin/* /usr/bin
 
 # Adding this step after copying the ceramic-one binary so that we always take the newest libs from the builder if the

Makefile

@@ -21,7 +21,9 @@ DEPLOY_TAG ?= latest
 # Whether or not this is a manual deployment
 MANUAL_DEPLOY ?= false
 
-DATABASE_URL ?= sqlite://checkpointer.db
+RUST_LOG ?= info
+
+.EXPORT_ALL_VARIABLES:
 
 .PHONY: all
 all: build check-fmt check-clippy test
@@ -57,9 +59,9 @@ test:
 	# Setup scaffolding
 	./ci-scripts/setup_test_env.sh
 	# Test with default features
-	DATABASE_URL=$(DATABASE_URL) $(CARGO) test -p checkpointer --locked --release
+	RUST_LOG=$(RUST_LOG) $(CARGO) test -p checkpointer --locked --release
 	# Test with all features
-	DATABASE_URL=$(DATABASE_URL) $(CARGO) test -p checkpointer --locked --release --all-features
+	RUST_LOG=$(RUST_LOG) $(CARGO) test -p checkpointer --locked --release --all-features
 	./ci-scripts/teardown_test_env.sh
 
 .PHONY: test-event-joiner
@@ -87,8 +89,14 @@ check-clippy:
 run:
 	RUST_LOG=WARN,checkpointer=DEBUG $(CARGO) run --all-features --locked --release --bin checkpointer
 
+.PHONY: build-docker
+build-docker:
+	./ci-scripts/setup_test_env.sh
+	./ci-scripts/image.sh
+	./ci-scripts/teardown_test_env.sh
+
 .PHONY: publish-docker
-publish-docker:
+publish-docker: build-docker
 	./ci-scripts/publish.sh
 
 .PHONY: schedule-ecs-deployment

checkpointer/Cargo.toml

@@ -13,6 +13,7 @@ async-trait = "0.1.77"
 calculator = { path = "../calculator" }
 ceramic-http-client = { workspace = true, features = ["remote"] }
 chrono.workspace = true
+clap.workspace = true
 futures-util = "0.3.30"
 models = { path = "../models" }
 reqwest = "0.11.23"
@@ -29,6 +30,6 @@ url.workspace = true
 util = { path = "../util" }
 
 [dev-dependencies]
-curve25519-dalek = "=4.1.1"
+env_logger = "0.11.2"
 tmpdir = "1.0.0"
 wiremock = "0.6.0-rc.2"

checkpointer/src/batcher.rs

@@ -290,7 +290,7 @@ mod tests {
 
     #[tokio::test]
     async fn should_receive_create_and_update_events() {
-        let _guard = util::init_tracing();
+        let _ = env_logger::try_init();
 
         let ceramic_url = ceramic_url();
         let db = Arc::new(InMemoryPersistence::new());

checkpointer/src/errors.rs

@@ -22,6 +22,8 @@ pub enum Error {
     #[error("{0}")]
     Ceramic(#[from] anyhow::Error),
     #[error("{0}")]
+    Reqwest(#[from] reqwest::Error),
+    #[error("{0}")]
     Custom(String),
 }
 

checkpointer/src/main.rs

@@ -16,10 +16,25 @@ mod persistence;
 use crate::persistence::SqlitePersistence;
 use batcher::{BatchCreationParameters, Batcher};
 use calculator::CalculatorParameters;
+use clap::{Parser, Subcommand};
 use errors::Error;
 use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
+#[derive(Parser)]
+#[command(name = "CeramicCheckpointer")]
+#[command(version = "1.0")]
+#[command(about = "Provides batching and checkpointing for ceramic sse feeds", long_about = None)]
+struct Cli {
+    #[clap(subcommand)]
+    subcmd: Option<SubCmd>,
+}
+
+#[derive(Subcommand)]
+enum SubCmd {
+    SshCheck,
+}
+
 fn trace_error<B>(res: ServiceResponse<B>) -> Result<ErrorHandlerResponse<B>> {
     if let Some(ref e) = res.response().error() {
         tracing::error!("{:?}", e);
@@ -106,13 +121,32 @@ pub struct Config {
 #[actix_web::main]
 async fn main() -> Result<(), Error> {
     let _guard = util::init_tracing();
+    let cmd = Cli::parse();
 
-    let config = Config {
-        batcher: Batcher::new(Arc::new(SqlitePersistence::new().await?))?,
-        calculator_params: CalculatorParameters::new().await?,
-        calculate_active: Arc::new(AtomicBool::new(false)),
-    };
+    let calculator_params = CalculatorParameters::new().await?;
+    match cmd.subcmd {
+        Some(SubCmd::SshCheck) => {
+            let url = calculator_params
+                .ceramic_url
+                .join("/api/v0/node/healthcheck")?;
+            if !reqwest::get(url).await?.status().is_success() {
+                return Err(Error::custom("Failed to connect to ceramic"));
+            }
+        }
+        None => {
+            let config = Config {
+                batcher: Batcher::new(Arc::new(SqlitePersistence::new().await?))?,
+                calculator_params,
+                calculate_active: Arc::new(AtomicBool::new(false)),
+            };
+            start_server(config).await?;
+        }
+    }
+
+    Ok(())
+}
 
+async fn start_server(config: Config) -> Result<(), Error> {
     HttpServer::new(move || {
         let svc = web::scope("/api/v1")
             .service(create_batcher)

checkpointer/src/persistence.rs

@@ -108,6 +108,8 @@ pub mod tests {
 
     #[tokio::test]
     async fn can_add_and_retrieve_events() {
+        let _ = env_logger::try_init();
+
         let pool = setup().await;
         let client_id = "test_client";
         let event = Event {

ci-scripts/image.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ -z "$DID_DOCUMENT" ]; then
+  echo "No DID_DOCUMENT specified, cannot verify image"
+  exit 1
+fi
+
+# Build a docker image running checkpointer
+docker buildx build --load -t 3box/checkpointer .
+docker run -e DID_DOCUMENT=$DID_DOCUMENT -e DID_PRIVATE_KEY=$DID_PRIVATE_KEY -e CERAMIC_URL=$CERAMIC_URL --rm 3box/checkpointer ssh-check

ci-scripts/publish.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Build and publish a docker image run running checkpointer
+# Publish a docker image running checkpointer
 #
 # DOCKER_PASSWORD must be set
 # Use:
@@ -9,8 +9,6 @@
 #   echo "${DOCKER_PASSWORD}" | docker login --username AWS --password-stdin public.ecr.aws/r5b3e0r5
 #
 # to login to docker. That password will be valid for 12h.
-
-docker buildx build --load -t 3box/checkpointer .
 docker tag 3box/checkpointer:latest public.ecr.aws/r5b3e0r5/3box/checkpointer:latest
 
 if [[ -n "$SHA" ]]; then

ci-scripts/setup_test_env.sh

@@ -1,13 +1,18 @@
 #!/usr/bin/env bash
 docker compose -f it/docker-compose.yml up -d ceramic
 
+if [ -z "$DID_DOCUMENT" ]; then
+  echo "No DID_DOCUMENT specified, cannot setup environment"
+  exit 1
+fi
+
 echo "Starting ceramic"
 while [ $(curl -s -o /dev/null -I -w "%{http_code}" "http://localhost:7007/api/v0/node/healthcheck") -ne "200" ]; do
   echo "Ceramic is not yet ready, waiting and trying again"
   sleep 1
 done
 
-if [ -z "$IT_TEST_CHECKPOINTER" ]; then
+if [ -n "${IT_TEST_CHECKPOINTER}" ]; then
   echo "Starting Checkpointer"
   mkdir it/sqlite
   docker compose -f it/docker-compose.yml up -d checkpointer

it/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3'
 services:
   ceramic:
-    image: ceramicnetwork/js-ceramic:dev
+    image: ceramicnetwork/js-ceramic:5.2.0
     volumes:
       - ./data:/root/.ceramic
     ports:
@@ -15,6 +15,7 @@ services:
       - RUST_LOG=info
       - RUST_BACKTRACE=1
       - CERAMIC_URL=http://ceramic-service:7007
+      - DID_DOCUMENT=${DID_DOCUMENT}
       - DID_PRIVATE_KEY=${DID_PRIVATE_KEY}
       - DATABASE_URL=${DATABASE_URL}
     depends_on:

tester/Cargo.toml

@@ -9,8 +9,7 @@ edition = "2021"
 anyhow = "1.0.79"
 ceramic-http-client = { workspace = true, features = ["remote"] }
 chrono.workspace = true
-clap = { version = "=4.4.18", default-features = false, features = ["derive", "std"] }
-curve25519-dalek = "=4.1.1"
+clap.workspace = true
 models = { path = "../models" }
 serde.workspace = true
 serde_json.workspace = true