Staging release: Rust widget renderer

.cfignore

@@ -41,3 +41,4 @@
 # Ignore Rust build artifacts
 target/
 ext/widget_renderer/target/
+!ext/widget_renderer/libwidget_renderer.so

.circleci/config.yml

@@ -4,10 +4,15 @@
 #
 version: 2.1
 
+# Cancel redundant builds when new commits are pushed
+# This prevents multiple pipelines from racing to deploy
+# Note: This must also be enabled in CircleCI project settings
+# Settings > Advanced > Auto-cancel redundant builds
+
 jobs:
   build:
     docker:
-       - image: cimg/ruby:3.4.7-browsers # Updated to match Gemfile Ruby version
+       - image: cimg/ruby:3.2.8-browsers # Matches deployed Ruby version in CF
          environment:
             RAILS_ENV: test
             PGHOST: 127.0.0.1
@@ -41,6 +46,51 @@ jobs:
 
       - checkout
 
+      - run:
+          name: Install Rust toolchain
+          command: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+            echo 'source $HOME/.cargo/env' >> $BASH_ENV
+            source $HOME/.cargo/env
+            rustc --version
+            cargo --version
+
+      - restore_cache:
+          keys:
+            - v4-cargo-{{ checksum "ext/widget_renderer/Cargo.lock" }}-{{ checksum "ext/widget_renderer/src/template_renderer.rs" }}
+            - v4-cargo-{{ checksum "ext/widget_renderer/Cargo.lock" }}-
+
+      - run:
+          name: Build widget renderer (Rust)
+          command: |
+            source $HOME/.cargo/env
+            cargo build --release --manifest-path ext/widget_renderer/Cargo.toml
+
+      - run:
+          name: Verify Rust native library linkage
+          command: |
+            set -euo pipefail
+            LIB=ext/widget_renderer/target/release/libwidget_renderer.so
+            if [ -f "$LIB" ]; then
+              echo "Found built rust library; verifying linkage..."
+              if ldd "$LIB" 2>&1 | grep -q "not found"; then
+                echo "ERROR: Rust library has unresolved dependencies (ldd shows 'not found')."
+                ldd "$LIB" || true
+                exit 1
+              else
+                echo "Rust library linkage looks good"
+              fi
+            else
+              echo "No Rust library built - skipping linkage verification"
+            fi
+
+      - save_cache:
+          paths:
+            - ext/widget_renderer/target
+            - ~/.cargo/registry
+            - ~/.cargo/git
+          key: v4-cargo-{{ checksum "ext/widget_renderer/Cargo.lock" }}-{{ checksum "ext/widget_renderer/src/template_renderer.rs" }}
+
       # Download and cache dependencies
       - restore_cache:
           keys:
@@ -71,17 +121,20 @@ jobs:
             mkdir /tmp/test-results
             TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
 
-            bundle exec rspec --format progress \
-                            --format RspecJunitFormatter \
-                            --out /tmp/test-results/rspec.xml \
-                            --format progress \
-                            $TEST_FILES
+             bundle exec rspec --format progress \
+                             --format RspecJunitFormatter \
+                             --out /tmp/test-results/rspec.xml \
+                             --format progress \
+                             $TEST_FILES
       # Install Cloud Foundry cli (cf) before deploy step. cf is used to push to Cloud.gov
       - run:
           name: Install-cf-cli
           command: |
-            curl -v -L -o cf-cli_amd64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&source=github'
-            sudo dpkg -i cf-cli_amd64.deb
+            # Download and convert the GPG key to binary format for modern apt
+            wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo gpg --dearmor -o /usr/share/keyrings/cloudfoundry-cli.gpg
+            echo "deb [signed-by=/usr/share/keyrings/cloudfoundry-cli.gpg] https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+            sudo apt-get update
+            sudo apt-get install -y cf8-cli
             cf -v
       # collect reports
       - store_test_results:
@@ -91,11 +144,36 @@ jobs:
 
       - run:
           name: Deploy Sidekiq worker servers
-          command: ./.circleci/deploy-sidekiq.sh
+          command: |
+            # Only deploy from a single parallel node to avoid concurrent CF pushes.
+            if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then
+              echo "Skipping Sidekiq deploy on parallel node ${CIRCLE_NODE_INDEX}"
+              exit 0
+            fi
+            # Keep prebuilt Rust library - extconf.rb builds it during bundle install with correct paths
+            # The library is built with rutie which properly links against the CF Ruby installation
+            # echo "Removing prebuilt Rust library (will be rebuilt on CF)..."
+            # rm -rf ext/widget_renderer/target/release/libwidget_renderer.so 2>/dev/null || true
+            # rm -f ext/widget_renderer/libwidget_renderer.so 2>/dev/null || true
+            ./.circleci/deploy-sidekiq.sh
+          no_output_timeout: 30m
 
       - run:
           name: Deploy web server(s)
-          command: ./.circleci/deploy.sh
+          command: |
+            # Only deploy from a single parallel node to avoid concurrent CF pushes.
+            if [ "${CIRCLE_NODE_INDEX:-0}" != "0" ]; then
+              echo "Skipping web deploy on parallel node ${CIRCLE_NODE_INDEX}"
+              exit 0
+            fi
+            # Wait for Sidekiq deployment to complete before starting web deploy
+            sleep 120
+            # Keep prebuilt Rust library - extconf.rb builds it during bundle install with correct paths
+            # echo "Removing prebuilt Rust library (will be rebuilt on CF)..."
+            # rm -rf ext/widget_renderer/target/release/libwidget_renderer.so 2>/dev/null || true
+            # rm -f ext/widget_renderer/libwidget_renderer.so 2>/dev/null || true
+            ./.circleci/deploy.sh
+          no_output_timeout: 30m
 
   cron_tasks:
     docker:
@@ -105,8 +183,11 @@ jobs:
       - run:
           name: Install-cf-cli
           command: |
-            curl -v -L -o cf-cli_amd64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&source=github'
-            sudo dpkg -i cf-cli_amd64.deb
+            # Download and convert the GPG key to binary format for modern apt
+            wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo gpg --dearmor -o /usr/share/keyrings/cloudfoundry-cli.gpg
+            echo "deb [signed-by=/usr/share/keyrings/cloudfoundry-cli.gpg] https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+            sudo apt-get update
+            sudo apt-get install -y cf8-cli
             cf -v
       - run:
           name: Run CRON tasks

.circleci/deploy-sidekiq.sh

@@ -4,13 +4,176 @@
 # a non-zero exit code
 set -e
 
+# Acquire a deployment lock using CF environment variable
+# This prevents multiple pipelines from deploying simultaneously
+acquire_deploy_lock() {
+  local app_name="$1"
+  local lock_name="DEPLOY_LOCK"
+  local lock_value="${CIRCLE_BUILD_NUM:-$$}_$(date +%s)"
+  local max_wait=600  # 10 minutes max
+  local wait_interval=30
+  local waited=0
+
+  echo "Attempting to acquire deploy lock for $app_name..."
+
+  while [ $waited -lt $max_wait ]; do
+    # Check if there's an existing lock
+    local current_lock=$(cf env "$app_name" 2>/dev/null | grep "$lock_name:" | awk '{print $2}' || echo "")
+
+    if [ -z "$current_lock" ] || [ "$current_lock" == "null" ]; then
+      # No lock exists, try to acquire it
+      echo "Setting deploy lock: $lock_value"
+      cf set-env "$app_name" "$lock_name" "$lock_value" > /dev/null 2>&1 || true
+      sleep 5  # Small delay to handle race conditions
+
+      # Verify we got the lock
+      current_lock=$(cf env "$app_name" 2>/dev/null | grep "$lock_name:" | awk '{print $2}' || echo "")
+      if [ "$current_lock" == "$lock_value" ]; then
+        echo "Deploy lock acquired: $lock_value"
+        return 0
+      fi
+    fi
+
+    # Check if lock is stale (older than 15 minutes)
+    local lock_time=$(echo "$current_lock" | cut -d'_' -f2)
+    local now=$(date +%s)
+    if [ -n "$lock_time" ] && [ $((now - lock_time)) -gt 900 ]; then
+      echo "Stale lock detected (age: $((now - lock_time))s), clearing..."
+      cf unset-env "$app_name" "$lock_name" > /dev/null 2>&1 || true
+      continue
+    fi
+
+    echo "Deploy lock held by another process ($current_lock), waiting ${wait_interval}s... (waited ${waited}s)"
+    sleep $wait_interval
+    waited=$((waited + wait_interval))
+  done
+
+  echo "Warning: Could not acquire lock after ${max_wait}s, proceeding anyway..."
+  return 0
+}
+
+# Release the deployment lock
+release_deploy_lock() {
+  local app_name="$1"
+  local lock_name="DEPLOY_LOCK"
+  echo "Releasing deploy lock for $app_name..."
+  cf unset-env "$app_name" "$lock_name" > /dev/null 2>&1 || true
+}
+
+# Wait for any in-progress deployments to complete before starting
+wait_for_deployment() {
+  local app_name="$1"
+  local max_wait=600  # 10 minutes max
+  local wait_interval=15
+  local waited=0
+
+  echo "Checking for in-progress deployments of $app_name..."
+
+  while [ $waited -lt $max_wait ]; do
+    # Get deployment status - look for ACTIVE deployments
+    local status=$(cf curl "/v3/deployments?app_guids=$(cf app "$app_name" --guid)&status_values=ACTIVE" 2>/dev/null | grep -o '"state":"[^"]*"' | head -1 || echo "")
+
+    if [ -z "$status" ] || [[ "$status" == *'"state":"FINALIZED"'* ]] || [[ "$status" == *'"state":"DEPLOYED"'* ]]; then
+      echo "No active deployment in progress, proceeding..."
+      return 0
+    fi
+
+    echo "Deployment in progress ($status), waiting ${wait_interval}s... (waited ${waited}s of ${max_wait}s)"
+    sleep $wait_interval
+    waited=$((waited + wait_interval))
+  done
+
+  echo "Warning: Timed out waiting for previous deployment, proceeding anyway..."
+  return 0
+}
+
+# Retry function to handle staging and deployment conflicts
+cf_push_with_retry() {
+  local app_name="$1"
+  local max_retries=5
+  local retry_delay=90
+
+  # Acquire lock first
+  acquire_deploy_lock "$app_name"
+
+  # Ensure lock is released on exit
+  trap "release_deploy_lock '$app_name'" EXIT
+
+  # Wait for any in-progress deployment
+  wait_for_deployment "$app_name"
+
+  # Update app to use 180s invocation timeout and process health check before rolling deploy
+  echo "Updating health check configuration for $app_name..."
+  cf set-health-check "$app_name" process --invocation-timeout 180 || true
+  sleep 2
+
+  # Get current instance count and scale down to 1 to avoid memory quota issues during rolling deploy
+  echo "Checking current instance count for $app_name..."
+  local current_instances=$(cf app "$app_name" | grep "^instances:" | awk '{print $2}' | cut -d'/' -f2 || echo "1")
+  echo "Current instances: $current_instances"
+
+  if [ "$current_instances" -gt 1 ]; then
+    echo "Scaling down to 1 instance to free memory for rolling deploy..."
+    cf scale "$app_name" -i 1 || true
+    sleep 5
+  fi
+
+  for i in $(seq 1 $max_retries); do
+    echo "Attempt $i of $max_retries to push $app_name..."
+
+    # Stop the app first to free memory for staging
+    echo "Stopping $app_name to free memory for staging..."
+    cf stop "$app_name" || true
+    sleep 5
+
+    # Push without rolling strategy (direct replacement since we stopped it)
+    # Let CF auto-detect buildpacks to avoid re-running supply phase (Rust already built in CircleCI)
+    if cf push "$app_name" \
+      -t 180 \
+      --health-check-type process; then
+      echo "Successfully pushed $app_name"
+
+      # Scale back up to original instance count
+      if [ "$current_instances" -gt 1 ]; then
+        echo "Scaling up to $current_instances instances..."
+        cf scale "$app_name" -i "$current_instances" || true
+      fi
+
+      release_deploy_lock "$app_name"
+      trap - EXIT  # Clear the trap
+      return 0
+    else
+      local exit_code=$?
+      if [ $i -lt $max_retries ]; then
+        echo "Push failed (exit code: $exit_code), waiting ${retry_delay}s before retry..."
+        sleep $retry_delay
+        # Re-check for in-progress deployments before retrying
+        wait_for_deployment "$app_name"
+      fi
+    fi
+  done
+
+  # If we failed, try to scale back up anyway
+  if [ "$current_instances" -gt 1 ]; then
+    echo "Deploy failed, attempting to scale back up to $current_instances instances..."
+    cf scale "$app_name" -i "$current_instances" || true
+  fi
+
+  release_deploy_lock "$app_name"
+  trap - EXIT  # Clear the trap
+  echo "Failed to push $app_name after $max_retries attempts"
+  return 1
+}
+
 if [ "${CIRCLE_BRANCH}" == "production" ]
 then
   echo "Logging into cloud.gov"
   # Log into CF and push
   cf login -a $CF_API_ENDPOINT -u $CF_PRODUCTION_SPACE_DEPLOYER_USERNAME -p $CF_PRODUCTION_SPACE_DEPLOYER_PASSWORD -o $CF_ORG -s prod
   echo "PUSHING to PRODUCTION..."
-  cf push touchpoints-production-sidekiq-worker --strategy rolling
+  echo "Syncing Login.gov environment variables..."
+  ./.circleci/sync-login-gov-env.sh touchpoints-production-sidekiq-worker
+  cf_push_with_retry touchpoints-production-sidekiq-worker
   echo "Push to Production Complete."
 else
   echo "Not on the production branch."
@@ -22,7 +185,7 @@ then
   # Log into CF and push
   cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE
   echo "Pushing to Demo..."
-  cf push touchpoints-demo-sidekiq-worker --strategy rolling
+  cf_push_with_retry touchpoints-demo-sidekiq-worker
   echo "Push to Demo Complete."
 else
   echo "Not on the main branch."
@@ -34,7 +197,7 @@ then
   # Log into CF and push
   cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE
   echo "Pushing to Staging..."
-  cf push touchpoints-staging-sidekiq-worker --strategy rolling
+  cf_push_with_retry touchpoints-staging-sidekiq-worker
   echo "Push to Staging Complete."
 else
   echo "Not on the develop branch."