feat: use nightly releases of js-ceramic packages

Dockerfile

@@ -1,9 +1,7 @@
-FROM node:16-alpine as builder
+FROM node:20 as builder
 
 COPY package*.json ./
 
-RUN apk add --no-cache python3 make g++
-
 RUN npm ci --ignore-scripts
 
 # Hardcode the target arch for now
@@ -16,11 +14,11 @@ RUN npm rebuild loady
 RUN npm rebuild node-jq
 RUN npm rebuild sqlite3
 
-FROM node:16-alpine as app
+FROM node:20 as app
 
 WORKDIR /app
 
-RUN apk add --no-cache curl
+RUN apt update && apt install -y jq
 
 COPY --from=builder node_modules ./node_modules
 
@@ -38,7 +36,7 @@ ENV AWS_REGION=${AWS_REGION}
 ENV AWS_ECS_CLUSTER=${AWS_ECS_CLUSTER}
 ENV AWS_ECS_FAMILY=${AWS_ECS_FAMILY}
 ENV CERAMIC_ECS_CLUSTERS="ceramic-qa ceramic-qa-ex"
-ENV CERAMIC_URLS="https://ceramic-qa.3boxlabs.com https://gateway-qa.ceramic.network https://ceramic-private-qa.3boxlabs.com"
+ENV CERAMIC_URLS="https://ceramic-qa.3boxlabs.com https://ceramic-private-qa.3boxlabs.com"
 ENV CERAMIC_ENABLE_EXPERIMENTAL_COMPOSE_DB="true"
 
 # Discord notifications about running ECS tasks

README.md

@@ -50,9 +50,6 @@ CLOUDWATCH_LOG_BASE_URL=
 DISCORD_WEBHOOK_URL_TEST_FAILURES=
 DISCORD_WEBHOOK_URL_TEST_RESULTS=
 
-# `ceramic-utils-lambda` API Gateway IDs to fetch deployed commit hashes 
-APIGATEWAY_RESOURCE_ID=
-APIGATEWAY_RESTAPI_ID=
 ```
 Run the tests using the following commands:
 ```

ci/ci.sh

@@ -1,4 +1,8 @@
-#!/bin/sh
+#!/bin/bash
+
+# Run update script
+./update.sh
+
 export RUN_ID=$(node generate-id.js)
 echo "INFO: Run id: $RUN_ID"
 echo "INFO: Sleep seconds set for services to start: ${SLEEP:=60}"  # defaults to 60 seconds
@@ -17,7 +21,7 @@ if [[ $NODE_ENV == "local_client-public" ]]; then
 fi
 
 if [[ $NODE_ENV == "local_node-private" ]]; then
-  # init config and generate peer id 
+  # init config and generate peer id
   node_modules/go-ipfs/go-ipfs/ipfs init
 
   # config changes to match js-ceramic/ipfs-daemon: TODO remove these and ensure still passes
@@ -48,6 +52,6 @@ if [ "$EXPORT_LOGS" = true  ]; then
   node export-logs.js
 fi
 
-if [ $exit_code != 0 ] && [ "$REPORT_STATUS" = true]; then
-  node report-exit.js; 
+if [ $exit_code != 0 ] && [ "$REPORT_STATUS" = true ]; then
+  node report-exit.js;
 fi

ci/export-logs.js

@@ -9,7 +9,7 @@ AWS.config.update({
 })
 const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
 
-const directoryPath = path.join(import.meta.url, '../root/.ceramic/logs')
+const directoryPath = '/root/.ceramic/logs'
 
 function main() {
   fs.readdir(directoryPath, (err, files) => {

ci/helpers.js

@@ -1,68 +1,19 @@
 import * as https from 'https'
 import * as childProcess from 'child_process'
-import { ECSClient, ListTasksCommand } from '@aws-sdk/client-ecs'
+import {ECSClient, ListTasksCommand} from '@aws-sdk/client-ecs'
 
 // API gateway to Lambda to load the required clients and packages.
-import { APIGatewayClient, TestInvokeMethodCommand } from '@aws-sdk/client-api-gateway'
-
-const getCommitHashes = async () => {
-  try {
-    const client = new APIGatewayClient({ region: process.env.AWS_REGION })
-    const ceramicCmd = new TestInvokeMethodCommand({
-      restApiId: process.env.APIGATEWAY_RESTAPI_ID,
-      resourceId: process.env.APIGATEWAY_RESOURCE_ID,
-      httpMethod: 'GET',
-      pathWithQueryString: 'v1.0/infra?name=ceramic'
-    })
-    const ceramicResp = await client.send(ceramicCmd)
-    const ceramicJson = JSON.parse(ceramicResp.body)
-    const casCmd = new TestInvokeMethodCommand({
-      restApiId: process.env.APIGATEWAY_RESTAPI_ID,
-      resourceId: process.env.APIGATEWAY_RESOURCE_ID,
-      httpMethod: 'GET',
-      pathWithQueryString: 'v1.0/infra?name=cas'
-    })
-    const casResp = await client.send(casCmd)
-    const casJson = JSON.parse(casResp.body)
-    const ceramicDeployTag = ceramicJson.deployTag
-    const ceramicIpfsDeployTag = ceramicDeployTag
-    const casDeployTag = casJson.deployTag
-    const casIpfsDeployTag = casJson.buildInfo.sha_tag
-
-    const envUrls = `${process.env.CERAMIC_URLS}`.replace(/ /g, '\n')
-    const ceramicRepository = 'https://github.com/ceramicnetwork/js-ceramic'
-    const casRepository = 'https://github.com/ceramicnetwork/ceramic-anchor-service'
-    return `[js-ceramic (${ceramicDeployTag.substring(
-      0,
-      12
-    )})](${ceramicRepository}/commit/${ceramicDeployTag}) <==> [ipfs-daemon (${ceramicIpfsDeployTag.substring(
-      0,
-      12
-    )})](${ceramicRepository}/commit/${ceramicIpfsDeployTag})
-                      [ceramic-anchor-service (${casDeployTag.substring(
-                        0,
-                        12
-                      )})](${casRepository}/commit/${casDeployTag}) <==> [ipfs-daemon (${casIpfsDeployTag.substring(
-      0,
-      12
-    )})](${ceramicRepository}/commit/${casIpfsDeployTag})
-                      \`\`\`\n${envUrls}\`\`\` `
-  } catch (err) {
-    console.error(err)
-  }
-}
 
 /**
  * Returns the ARN for the running task
  * @returns {string}
  */
 function getThisTaskArn() {
-  const taskArn = childProcess
+  return childProcess
     .execSync(
       'curl -s "$ECS_CONTAINER_METADATA_URI_V4/task" | /app/node_modules/node-jq/bin/jq -r ".TaskARN" | awk -F/ \'{print $NF}\''
     )
     .toString()
-  return taskArn
 }
 
 /**
@@ -140,6 +91,5 @@ export {
   getThisTaskArn,
   generateDiscordCloudwatchLogUrl,
   listECSTasks,
-  sendDiscordNotification,
-  getCommitHashes
+  sendDiscordNotification
 }

ci/jest-reporter.js

@@ -2,18 +2,16 @@ import {
   getThisTaskArn,
   generateDiscordCloudwatchLogUrl,
   listECSTasks,
-  sendDiscordNotification,
-  getCommitHashes
+  sendDiscordNotification
 } from './helpers.js'
 import { BaseReporter } from '@jest/reporters'
 import * as childProcess from 'child_process'
 
 const userName = 'jest-reporter'
-let g_taskArns, g_commitHashes // g_ are global variables
+let g_taskArns // g_ are global variables
 
-async function listArntasksAndCommitHashes() {
+async function listArnTasks() {
   g_taskArns = await listECSTasks()
-  g_commitHashes = await getCommitHashes() // e.g. "ceramic-anchor-service (333fc9afb59a) <==> go-ipfs-daemon (6871b7dcd27d)\n"
 }
 
 export default class MyCustomReporter extends BaseReporter {
@@ -26,20 +24,17 @@ export default class MyCustomReporter extends BaseReporter {
   }
 
   onRunStart(results, options) {
-    listArntasksAndCommitHashes()
+    listArnTasks()
       .then(() => {
         console.log('INFO: onRunStart g_taskArns:=', g_taskArns)
-        console.log('INFO: onRunStart g_commitHashes:=', g_commitHashes)
-        this.commitHashes = g_commitHashes
         this.logUrl = generateDiscordCloudwatchLogUrl(this.taskArn)
         this.testFailuresUrl = process.env.DISCORD_WEBHOOK_URL_TEST_FAILURES
         this.testResultsUrl = process.env.DISCORD_WEBHOOK_URL_TEST_RESULTS
 
         const message = buildDiscordStartMessage(
           results,
           this.taskArn,
-          this.logUrl,
-          this.commitHashes
+          this.logUrl
         )
         const data = { embeds: message, username: userName }
 
@@ -56,8 +51,7 @@ export default class MyCustomReporter extends BaseReporter {
     const message = buildDiscordSummaryMessage(
       results,
       this.taskArn,
-      this.logUrl,
-      this.commitHashes
+      this.logUrl
     )
     const data = { embeds: message, username: userName }
 
@@ -85,7 +79,7 @@ export default class MyCustomReporter extends BaseReporter {
   }
 }
 
-function buildDiscordStartMessage(results, taskArn, logUrl, commitHashes) {
+function buildDiscordStartMessage(results, taskArn, logUrl) {
   let startedAt = results.startTime
   try {
     startedAt = new Date(results.startTime).toGMTString()
@@ -113,10 +107,6 @@ function buildDiscordStartMessage(results, taskArn, logUrl, commitHashes) {
           name: 'Started at',
           value: startedAt
         },
-        {
-          name: 'Commit hashes',
-          value: `${commitHashes}`
-        },
         {
           name: 'Logs',
           value: `${logUrl}`
@@ -127,7 +117,7 @@ function buildDiscordStartMessage(results, taskArn, logUrl, commitHashes) {
   return discordEmbeds
 }
 
-function buildDiscordSummaryMessage(results, taskArn, logUrl, commitHashes) {
+function buildDiscordSummaryMessage(results, taskArn, logUrl) {
   let startedAt = results.startTime
   try {
     startedAt = new Date(results.startTime).toGMTString()
@@ -177,10 +167,6 @@ function buildDiscordSummaryMessage(results, taskArn, logUrl, commitHashes) {
           name: 'Tests',
           value: `Passed: ${results.numPassedTests}, Failed: ${results.numFailedTests}, Total: ${results.numTotalTests}`
         },
-        {
-          name: 'Commit hashes',
-          value: `${commitHashes}`
-        },
         {
           name: 'Logs',
           value: `${logUrl}`

ci/report-exit.js

@@ -1,14 +1,12 @@
 import {
   generateDiscordCloudwatchLogUrl,
   listECSTasks,
-  sendDiscordNotification,
-  getCommitHashes
+  sendDiscordNotification
 } from './helpers.js'
 
 const main = async () => {
   try {
     const taskArns = await listECSTasks()
-    const commitHashes = await getCommitHashes() // e.g. "ceramic-anchor-service (333fc9afb59a) <==> go-ipfs-daemon (6871b7dcd27d)\n"
     console.log('INFO: listECSTasks taskArns:=', taskArns)
     let logUrls = generateDiscordCloudwatchLogUrl()
     if (logUrls.length < 1) {
@@ -25,10 +23,6 @@ const main = async () => {
             name: 'Configuration',
             value: `${process.env.NODE_ENV}`
           },
-          {
-            name: 'Commit hashes',
-            value: `${commitHashes}`
-          },
           {
             name: 'Logs',
             value: `${logUrls}`

ci/update.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+deps=$(jq -r '.dependencies | keys | .[]' package.json)
+filtered_deps=''
+for dep in $deps
+do
+    # Look for specific deps we manage
+    case $dep in
+        "dids" | "key-did-provider-ed25519" | "key-did-resolver")
+            filtered_deps="$filtered_deps $dep@latest"
+            continue
+            ;;
+        *)
+            # Ignore other deps
+            ;;
+    esac
+    scope=$(dirname "$dep")
+    case $scope in
+        "@ceramicnetwork"|"@composedb")
+            filtered_deps="$filtered_deps $dep@nightly"
+            ;;
+        *)
+            # Ignore other deps
+            ;;
+    esac
+done
+
+cmd="npm install --ignore-scripts $filtered_deps"
+echo "$cmd"
+$cmd
+
+# We have to repeat this step after updating dependencies
+cmd="npm run postinstall -prefix ./node_modules/go-ipfs"
+echo "$cmd"
+$cmd
+
+echo "Updated versions of dependencies:"
+jq -r '.dependencies' package.json