revert: restore Fly.io infrastructure, revert Electric Cloud migration

.github/templates/preview-comment.md

@@ -14,6 +14,11 @@
 <td>$DATABASE_LINK</td>
 </tr>
 <tr>
+<td><img src="https://fly.io/phx/ui/images/favicon/favicon-595d1312b35dfe32838befdf8505515e.ico" width="20" height="20" alt="Fly.io"> <strong>Electric (Fly.io)</strong></td>
+<td align="center">$ELECTRIC_STATUS</td>
+<td>$ELECTRIC_LINK</td>
+</tr>
+<tr>
 <td><img src="https://vercel.com/favicon.ico" width="20" height="20" alt="Vercel"> <strong>API (Vercel)</strong></td>
 <td align="center">$API_STATUS</td>
 <td>$API_LINK</td>

.github/workflows/cleanup-preview.yml

@@ -22,6 +22,17 @@ jobs:
           branch: ${{ github.event.pull_request.head.ref }}
           api_key: ${{ secrets.NEON_API_KEY }}
 
+      - name: Setup Fly CLI
+        uses: superfly/flyctl-actions/setup-flyctl@master
+
+      - name: Delete Electric Fly.io app
+        id: electric-cleanup
+        continue-on-error: true
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        run: |
+          flyctl apps destroy "superset-electric-pr-${{ github.event.pull_request.number }}" --yes
+
       - name: Update comment
         if: always()
         uses: thollander/actions-comment-pull-request@v3
@@ -31,6 +42,7 @@ jobs:
 
             The following preview resources have been cleaned up:
             - ${{ steps.neon-cleanup.outcome == 'success' && '✅' || '⚠️' }} Neon database branch
+            - ${{ steps.electric-cleanup.outcome == 'success' && '✅' || '⚠️' }} Electric Fly.io app
 
             Thank you for your contribution! 🎉
           comment-tag: "🚀-preview-deployment"

.github/workflows/deploy-preview.yml

@@ -17,6 +17,7 @@ env:
   MARKETING_ALIAS: marketing-pr-${{ github.event.pull_request.number }}-superset.vercel.app
   ADMIN_ALIAS: admin-pr-${{ github.event.pull_request.number }}-superset.vercel.app
   DOCS_ALIAS: docs-pr-${{ github.event.pull_request.number }}-superset.vercel.app
+  ELECTRIC_URL: https://superset-electric-pr-${{ github.event.pull_request.number }}.fly.dev/v1/shape
 
 jobs:
   deploy-database:
@@ -74,6 +75,51 @@ jobs:
           name: database-status
           path: database-status.env
 
+  deploy-electric:
+    name: Deploy Electric (Fly.io)
+    runs-on: ubuntu-latest
+    needs: deploy-database
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download database info
+        uses: actions/download-artifact@v4
+        with:
+          name: database-status
+
+      - name: Load database URL
+        run: |
+          source database-status.env
+          echo "DATABASE_URL_UNPOOLED=$DATABASE_URL_UNPOOLED" >> $GITHUB_ENV
+
+      - name: Deploy Electric to Fly.io
+        uses: superfly/fly-pr-review-apps@1.3.0
+        with:
+          name: superset-electric-pr-${{ github.event.pull_request.number }}
+          region: iad
+          org: ${{ vars.FLY_ORG }}
+          config: fly.toml
+          secrets: |
+            DATABASE_URL=${{ env.DATABASE_URL_UNPOOLED }}
+            ELECTRIC_SECRET=${{ secrets.ELECTRIC_SECRET_PREVIEW }}
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+
+      - name: Save Electric status
+        run: |
+          cat > electric-status.env << EOF
+          ELECTRIC_STATUS="✅"
+          ELECTRIC_LINK="<a href=\"https://fly.io/apps/superset-electric-pr-${{ github.event.pull_request.number }}\">View App</a>"
+          EOF
+
+      - name: Upload Electric status
+        uses: actions/upload-artifact@v4
+        with:
+          name: electric-status
+          path: electric-status.env
+
   deploy-api:
     name: Deploy API
     runs-on: ubuntu-latest
@@ -153,8 +199,8 @@ jobs:
           QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }}
           QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }}
           QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }}
-          ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }}
-          ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }}
+          ELECTRIC_URL: ${{ env.ELECTRIC_URL }}
+          ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET_PREVIEW }}
           DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }}
           DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }}
           STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
@@ -202,8 +248,8 @@ jobs:
             --env QSTASH_TOKEN=$QSTASH_TOKEN \
             --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \
             --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \
-            --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \
-            --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \
+            --env ELECTRIC_URL=$ELECTRIC_URL \
+            --env ELECTRIC_SECRET=$ELECTRIC_SECRET \
             --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \
             --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \
             --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \
@@ -625,7 +671,7 @@ jobs:
     name: Post Deployment Comment
     runs-on: ubuntu-latest
     if: always()
-    needs: [deploy-database, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs]
+    needs: [deploy-database, deploy-electric, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs]
     permissions:
       contents: read
       pull-requests: write
@@ -644,6 +690,8 @@ jobs:
         run: |
           DATABASE_STATUS="❌"
           DATABASE_LINK="Failed to create"
+          ELECTRIC_STATUS="❌"
+          ELECTRIC_LINK="Failed to deploy"
           API_STATUS="❌"
           API_LINK="Failed to deploy"
           WEB_STATUS="❌"
@@ -659,6 +707,10 @@ jobs:
             source database-status.env
           fi
 
+          if [[ "${{ needs.deploy-electric.result }}" == "success" ]]; then
+            source electric-status.env
+          fi
+
           if [[ "${{ needs.deploy-api.result }}" == "success" ]]; then
             source api-status.env
           fi
@@ -679,7 +731,7 @@ jobs:
             source docs-status.env
           fi
 
-          export DATABASE_STATUS DATABASE_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK
+          export DATABASE_STATUS DATABASE_LINK ELECTRIC_STATUS ELECTRIC_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK
           envsubst < .github/templates/preview-comment.md > final-comment.md
 
       - name: Post final deployment comment

.github/workflows/deploy-production.yml

@@ -103,8 +103,8 @@ jobs:
           QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }}
           QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }}
           QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }}
-          ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }}
-          ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }}
+          ELECTRIC_URL: ${{ secrets.ELECTRIC_URL }}
+          ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET }}
           DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }}
           DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }}
           STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
@@ -152,8 +152,8 @@ jobs:
             --env QSTASH_TOKEN=$QSTASH_TOKEN \
             --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \
             --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \
-            --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \
-            --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \
+            --env ELECTRIC_URL=$ELECTRIC_URL \
+            --env ELECTRIC_SECRET=$ELECTRIC_SECRET \
             --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \
             --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \
             --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \
@@ -418,6 +418,33 @@ jobs:
             --env SECRETS_ENCRYPTION_KEY=$SECRETS_ENCRYPTION_KEY \
             --env ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY
 
+  deploy-electric:
+    name: Deploy Electric to Fly.io
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Fly CLI
+        uses: superfly/flyctl-actions/setup-flyctl@master
+
+      - name: Stage secrets
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        run: |
+          flyctl secrets set \
+            DATABASE_URL="${{ secrets.DATABASE_URL_UNPOOLED }}" \
+            ELECTRIC_SECRET="${{ secrets.ELECTRIC_SECRET }}" \
+            --app superset-electric \
+            --stage
+
+      - name: Deploy to Fly.io
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        run: flyctl deploy . --config fly.toml --remote-only
+
   deploy-docs:
     name: Deploy Docs to Vercel
     runs-on: ubuntu-latest

apps/api/src/app/api/electric/[...path]/route.ts

@@ -24,26 +24,22 @@ export async function GET(request: Request): Promise<Response> {
 		return new Response("Not a member of this organization", { status: 403 });
 	}
 
-	const {
-		ELECTRIC_SOURCE_ID,
-		ELECTRIC_SOURCE_SECRET,
-		ELECTRIC_URL,
-		ELECTRIC_SECRET,
-	} = env;
-
-	let originUrl: URL;
-	if (ELECTRIC_SOURCE_ID && ELECTRIC_SOURCE_SECRET) {
-		originUrl = new URL("/v1/shape", "https://api.electric-sql.cloud");
-		originUrl.searchParams.set("source_id", ELECTRIC_SOURCE_ID);
-		originUrl.searchParams.set("source_secret", ELECTRIC_SOURCE_SECRET);
-	} else if (ELECTRIC_URL && ELECTRIC_SECRET) {
-		originUrl = new URL(ELECTRIC_URL);
-		originUrl.searchParams.set("secret", ELECTRIC_SECRET);
+	const useCloud =
+		request.headers.get("x-electric-backend") === "cloud" &&
+		env.ELECTRIC_SOURCE_ID &&
+		env.ELECTRIC_SOURCE_SECRET;
+
+	const originUrl = useCloud
+		? new URL("/v1/shape", "https://api.electric-sql.cloud")
+		: new URL(env.ELECTRIC_URL);
+
+	if (useCloud) {
+		// biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above
+		originUrl.searchParams.set("source_id", env.ELECTRIC_SOURCE_ID!);
+		// biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above
+		originUrl.searchParams.set("source_secret", env.ELECTRIC_SOURCE_SECRET!);
 	} else {
-		return new Response(
-			"Missing Electric config: set ELECTRIC_SOURCE_ID/SECRET or ELECTRIC_URL/SECRET",
-			{ status: 500 },
-		);
+		originUrl.searchParams.set("secret", env.ELECTRIC_SECRET);
 	}
 
 	url.searchParams.forEach((value, key) => {
@@ -89,7 +85,7 @@ export async function GET(request: Request): Promise<Response> {
 	const response = await fetch(originUrl.toString());
 
 	const headers = new Headers(response.headers);
-	headers.append("Vary", "Authorization");
+	headers.append("Vary", "Authorization, X-Electric-Backend");
 
 	if (headers.get("content-encoding")) {
 		headers.delete("content-encoding");

apps/api/src/env.ts

@@ -10,10 +10,10 @@ export const env = createEnv({
 	server: {
 		DATABASE_URL: z.string(),
 		DATABASE_URL_UNPOOLED: z.string(),
+		ELECTRIC_URL: z.string().url(),
+		ELECTRIC_SECRET: z.string().min(16),
 		ELECTRIC_SOURCE_ID: z.string().optional(),
 		ELECTRIC_SOURCE_SECRET: z.string().optional(),
-		ELECTRIC_URL: z.string().url().optional(),
-		ELECTRIC_SECRET: z.string().optional(),
 		BLOB_READ_WRITE_TOKEN: z.string(),
 		GOOGLE_CLIENT_ID: z.string().min(1),
 		GOOGLE_CLIENT_SECRET: z.string().min(1),

apps/api/src/proxy.ts

@@ -24,7 +24,7 @@ function getCorsHeaders(origin: string | null) {
 		"Access-Control-Allow-Origin": isAllowed ? origin : "",
 		"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
 		"Access-Control-Allow-Headers":
-			"Content-Type, Authorization, x-trpc-source, trpc-accept, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed",
+			"Content-Type, Authorization, x-trpc-source, trpc-accept, X-Electric-Backend, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed",
 		"Access-Control-Expose-Headers": [
 			// Electric sync headers
 			"electric-offset",

apps/desktop/electron.vite.config.ts

@@ -77,7 +77,7 @@ export default defineConfig({
 			),
 			"process.env.STREAMS_URL": defineEnv(
 				process.env.STREAMS_URL,
-				"https://streams.superset.sh",
+				"https://superset-stream.fly.dev",
 			),
 			"process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT),
 			"process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv(
@@ -178,7 +178,7 @@ export default defineConfig({
 			),
 			"process.env.STREAMS_URL": defineEnv(
 				process.env.STREAMS_URL,
-				"https://streams.superset.sh",
+				"https://superset-stream.fly.dev",
 			),
 			"process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT),
 			"process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv(

apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts

@@ -82,6 +82,7 @@ const electricHeaders = {
 		const token = getAuthToken();
 		return token ? `Bearer ${token}` : "";
 	},
+	"X-Electric-Backend": "cloud",
 };
 
 const organizationsCollection = createCollection(

fly.toml

@@ -0,0 +1,32 @@
+app = "superset-electric"
+primary_region = "iad"
+
+[build]
+image = "electricsql/electric:1.4.3"
+
+[[vm]]
+memory = "8192mb"
+cpu_kind = "performance"
+cpus = 4
+
+[env]
+ELECTRIC_DATABASE_USE_IPV6 = "true"
+ELECTRIC_MAX_CONCURRENT_REQUESTS = '{"initial": 3000, "existing": 10000}'
+
+[http_service]
+internal_port = 3000
+force_https = true
+auto_stop_machines = "off"
+auto_start_machines = true
+min_machines_running = 1
+
+[[http_service.checks]]
+interval = "10s"
+timeout = "2s"
+grace_period = "20s"
+method = "GET"
+path = "/v1/health"
+
+[mounts]
+source = "electric_data"
+destination = "/var/lib/electric"