Redis failover detection on app launch

src/api/index.ts

@@ -24,7 +24,6 @@ import {
 } from "fastify-zod-openapi";
 import { type ZodOpenApiVersion } from "zod-openapi";
 import { withTags } from "./components/index.js";
-import RedisModule from "ioredis";
 
 /** BEGIN EXTERNAL PLUGINS */
 import fastifyIp from "fastify-ip";
@@ -61,6 +60,7 @@ import mobileWalletV2Route from "./routes/v2/mobileWallet.js";
 import membershipV2Plugin from "./routes/v2/membership.js";
 import { docsHtml, securitySchemes } from "./docs.js";
 import syncIdentityPlugin from "./routes/syncIdentity.js";
+import { createRedisModule } from "./redis.js";
 /** END ROUTES */
 
 export const instanceId = randomUUID();
@@ -309,7 +309,11 @@ Otherwise, email [[email protected]](mailto:[email protected]) for sup
       ) as SecretConfig;
     };
     await app.refreshSecretConfig();
-    app.redisClient = new RedisModule.default(app.secretConfig.redis_url);
+    app.redisClient = await createRedisModule(
+      app.secretConfig.redis_url,
+      app.secretConfig.fallback_redis_url,
+      app.log,
+    );
   }
   if (isRunningInLambda) {
     await app.register(fastifyIp.default, {

src/api/redis.ts

@@ -0,0 +1,76 @@
+import RedisModule from "ioredis";
+import { ValidLoggers } from "./types.js";
+
+type RedisMode = "read-write" | "read-only" | "down";
+
+async function checkRedisMode(url: string): Promise<RedisMode> {
+  let testModule: RedisModule.default | null = null;
+  try {
+    testModule = new RedisModule.default(url);
+
+    // Test connectivity
+    await testModule.ping();
+
+    // Test write capability
+    const testKey = `health-check:${Date.now()}`;
+    try {
+      await testModule.set(testKey, "test", "EX", 1);
+      return "read-write";
+    } catch (writeError) {
+      return "read-only";
+    }
+  } catch (error) {
+    return "down";
+  } finally {
+    if (testModule) {
+      await testModule.quit().catch(() => {});
+    }
+  }
+}
+
+export async function createRedisModule(
+  primaryUrl: string,
+  fallbackUrl: string,
+  logger: ValidLoggers,
+) {
+  const primaryMode = await checkRedisMode(primaryUrl);
+  if (Math.random() < 0.01) {
+    // Upstash will complain if we never hit the instance
+    // Fire a hit every so often on the fallback
+    // Don't block on it
+    // Given we create a client once ever hour, we should hit the node at least once every 4 days at least
+    checkRedisMode(fallbackUrl);
+  }
+  if (primaryMode === "read-write") {
+    logger.info("Using primary Redis in read-write mode");
+    return new RedisModule.default(primaryUrl);
+  }
+
+  const fallbackMode = await checkRedisMode(fallbackUrl);
+
+  if (fallbackMode === "read-write") {
+    logger.warn(
+      `Primary Redis is ${primaryMode}, using fallback in read-write mode`,
+    );
+    return new RedisModule.default(fallbackUrl);
+  }
+
+  if (primaryMode === "read-only") {
+    logger.warn(
+      "Both Redis instances are read-only. Using primary in read-only mode",
+    );
+    return new RedisModule.default(primaryUrl);
+  }
+
+  if (fallbackMode === "read-only") {
+    logger.error(
+      "Primary Redis is down and Fallback Redis is down, using primary anyway",
+    );
+    return new RedisModule.default(primaryUrl);
+  }
+
+  logger.error(
+    "Both primary and fallback Redis instances are down. Creating client on primary anyway.",
+  );
+  return new RedisModule.default(primaryUrl);
+}

src/api/sqs/handlers/createOrgGithubTeam.ts

@@ -20,6 +20,7 @@ import { Modules } from "common/modules.js";
 import { retryDynamoTransactionWithBackoff } from "api/utils.js";
 import { SKIP_EXTERNAL_ORG_LEAD_UPDATE } from "common/overrides.js";
 import { getOrgByName } from "@acm-uiuc/js-shared";
+import { createRedisModule } from "api/redis.js";
 
 export const createOrgGithubTeamHandler: SQSHandlerFunction<
   AvailableSQSFunctions.CreateOrgGithubTeam
@@ -28,7 +29,11 @@ export const createOrgGithubTeamHandler: SQSHandlerFunction<
     logger,
     commonConfig: { region: genericConfig.AwsRegion },
   });
-  const redisClient = new RedisModule.default(secretConfig.redis_url);
+  const redisClient = await createRedisModule(
+    secretConfig.redis_url,
+    secretConfig.fallback_redis_url,
+    logger,
+  );
   try {
     const { orgName, githubTeamName, githubTeamDescription } = payload;
     const orgImmutableId = getOrgByName(orgName)!.id;

src/api/sqs/handlers/provisionNewMember.ts

@@ -13,6 +13,7 @@ import { getAuthorizedClients, getSecretConfig } from "../utils.js";
 import { emailMembershipPassHandler } from "./emailMembershipPassHandler.js";
 import RedisModule from "ioredis";
 import { setKey } from "api/functions/redisCache.js";
+import { createRedisModule } from "api/redis.js";
 
 export const provisionNewMemberHandler: SQSHandlerFunction<
   AvailableSQSFunctions.ProvisionNewMember
@@ -30,7 +31,11 @@ export const provisionNewMemberHandler: SQSHandlerFunction<
     logger,
     commonConfig,
   });
-  const redisClient = new RedisModule.default(secretConfig.redis_url);
+  const redisClient = await createRedisModule(
+    secretConfig.redis_url,
+    secretConfig.fallback_redis_url,
+    logger,
+  );
   const netId = email.replace("@illinois.edu", "");
   const cacheKey = `membership:${netId}:acmpaid`;
   logger.info("Got authorized clients and Entra ID token.");

src/common/config.ts

@@ -189,6 +189,7 @@ export type SecretConfig = {
   stripe_endpoint_secret: string;
   stripe_links_endpoint_secret: string;
   redis_url: string;
+  fallback_redis_url: string;
   encryption_key: string;
   UIN_HASHING_SECRET_PEPPER: string;
   github_pat: string;