diff --git a/.github/templates/preview-comment.md b/.github/templates/preview-comment.md index f555706ecd5..a1bd37c3d40 100644 --- a/.github/templates/preview-comment.md +++ b/.github/templates/preview-comment.md @@ -14,6 +14,11 @@ $DATABASE_LINK +Fly.io Electric (Fly.io) +$ELECTRIC_STATUS +$ELECTRIC_LINK + + Vercel API (Vercel) $API_STATUS $API_LINK diff --git a/.github/workflows/cleanup-preview.yml b/.github/workflows/cleanup-preview.yml index 09a6710df57..e7b4e732e26 100644 --- a/.github/workflows/cleanup-preview.yml +++ b/.github/workflows/cleanup-preview.yml @@ -22,6 +22,17 @@ jobs: branch: ${{ github.event.pull_request.head.ref }} api_key: ${{ secrets.NEON_API_KEY }} + - name: Setup Fly CLI + uses: superfly/flyctl-actions/setup-flyctl@master + + - name: Delete Electric Fly.io app + id: electric-cleanup + continue-on-error: true + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + run: | + flyctl apps destroy "superset-electric-pr-${{ github.event.pull_request.number }}" --yes + - name: Update comment if: always() uses: thollander/actions-comment-pull-request@v3 @@ -31,6 +42,7 @@ jobs: The following preview resources have been cleaned up: - ${{ steps.neon-cleanup.outcome == 'success' && '✅' || '⚠️' }} Neon database branch + - ${{ steps.electric-cleanup.outcome == 'success' && '✅' || '⚠️' }} Electric Fly.io app Thank you for your contribution! 🎉 comment-tag: "🚀-preview-deployment" diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index 749a58d7bc1..6f9bf4cfed7 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -17,6 +17,7 @@ env: MARKETING_ALIAS: marketing-pr-${{ github.event.pull_request.number }}-superset.vercel.app ADMIN_ALIAS: admin-pr-${{ github.event.pull_request.number }}-superset.vercel.app DOCS_ALIAS: docs-pr-${{ github.event.pull_request.number }}-superset.vercel.app + ELECTRIC_URL: https://superset-electric-pr-${{ github.event.pull_request.number }}.fly.dev/v1/shape jobs: deploy-database: @@ -74,6 +75,51 @@ jobs: name: database-status path: database-status.env + deploy-electric: + name: Deploy Electric (Fly.io) + runs-on: ubuntu-latest + needs: deploy-database + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download database info + uses: actions/download-artifact@v4 + with: + name: database-status + + - name: Load database URL + run: | + source database-status.env + echo "DATABASE_URL_UNPOOLED=$DATABASE_URL_UNPOOLED" >> $GITHUB_ENV + + - name: Deploy Electric to Fly.io + uses: superfly/fly-pr-review-apps@1.3.0 + with: + name: superset-electric-pr-${{ github.event.pull_request.number }} + region: iad + org: ${{ vars.FLY_ORG }} + config: fly.toml + secrets: | + DATABASE_URL=${{ env.DATABASE_URL_UNPOOLED }} + ELECTRIC_SECRET=${{ secrets.ELECTRIC_SECRET_PREVIEW }} + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + + - name: Save Electric status + run: | + cat > electric-status.env << EOF + ELECTRIC_STATUS="✅" + ELECTRIC_LINK="View App" + EOF + + - name: Upload Electric status + uses: actions/upload-artifact@v4 + with: + name: electric-status + path: electric-status.env + deploy-api: name: Deploy API runs-on: ubuntu-latest @@ -153,8 +199,8 @@ jobs: QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }} QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }} QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }} - ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }} - ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }} + ELECTRIC_URL: ${{ env.ELECTRIC_URL }} + ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET_PREVIEW }} DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }} DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }} STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} @@ -202,8 +248,8 @@ jobs: --env QSTASH_TOKEN=$QSTASH_TOKEN \ --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \ --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \ - --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \ - --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \ + --env ELECTRIC_URL=$ELECTRIC_URL \ + --env ELECTRIC_SECRET=$ELECTRIC_SECRET \ --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \ --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \ --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \ @@ -625,7 +671,7 @@ jobs: name: Post Deployment Comment runs-on: ubuntu-latest if: always() - needs: [deploy-database, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs] + needs: [deploy-database, deploy-electric, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs] permissions: contents: read pull-requests: write @@ -644,6 +690,8 @@ jobs: run: | DATABASE_STATUS="❌" DATABASE_LINK="Failed to create" + ELECTRIC_STATUS="❌" + ELECTRIC_LINK="Failed to deploy" API_STATUS="❌" API_LINK="Failed to deploy" WEB_STATUS="❌" @@ -659,6 +707,10 @@ jobs: source database-status.env fi + if [[ "${{ needs.deploy-electric.result }}" == "success" ]]; then + source electric-status.env + fi + if [[ "${{ needs.deploy-api.result }}" == "success" ]]; then source api-status.env fi @@ -679,7 +731,7 @@ jobs: source docs-status.env fi - export DATABASE_STATUS DATABASE_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK + export DATABASE_STATUS DATABASE_LINK ELECTRIC_STATUS ELECTRIC_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK envsubst < .github/templates/preview-comment.md > final-comment.md - name: Post final deployment comment diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 7f851aad7f9..55f2a233493 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -103,8 +103,8 @@ jobs: QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }} QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }} QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }} - ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }} - ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }} + ELECTRIC_URL: ${{ secrets.ELECTRIC_URL }} + ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET }} DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }} DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }} STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} @@ -152,8 +152,8 @@ jobs: --env QSTASH_TOKEN=$QSTASH_TOKEN \ --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \ --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \ - --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \ - --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \ + --env ELECTRIC_URL=$ELECTRIC_URL \ + --env ELECTRIC_SECRET=$ELECTRIC_SECRET \ --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \ --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \ --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \ @@ -418,6 +418,33 @@ jobs: --env SECRETS_ENCRYPTION_KEY=$SECRETS_ENCRYPTION_KEY \ --env ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY + deploy-electric: + name: Deploy Electric to Fly.io + runs-on: ubuntu-latest + environment: production + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Fly CLI + uses: superfly/flyctl-actions/setup-flyctl@master + + - name: Stage secrets + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + run: | + flyctl secrets set \ + DATABASE_URL="${{ secrets.DATABASE_URL_UNPOOLED }}" \ + ELECTRIC_SECRET="${{ secrets.ELECTRIC_SECRET }}" \ + --app superset-electric \ + --stage + + - name: Deploy to Fly.io + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + run: flyctl deploy . --config fly.toml --remote-only + deploy-docs: name: Deploy Docs to Vercel runs-on: ubuntu-latest diff --git a/apps/api/src/app/api/electric/[...path]/route.ts b/apps/api/src/app/api/electric/[...path]/route.ts index d434a2b6741..007fc8359ab 100644 --- a/apps/api/src/app/api/electric/[...path]/route.ts +++ b/apps/api/src/app/api/electric/[...path]/route.ts @@ -24,26 +24,22 @@ export async function GET(request: Request): Promise { return new Response("Not a member of this organization", { status: 403 }); } - const { - ELECTRIC_SOURCE_ID, - ELECTRIC_SOURCE_SECRET, - ELECTRIC_URL, - ELECTRIC_SECRET, - } = env; - - let originUrl: URL; - if (ELECTRIC_SOURCE_ID && ELECTRIC_SOURCE_SECRET) { - originUrl = new URL("/v1/shape", "https://api.electric-sql.cloud"); - originUrl.searchParams.set("source_id", ELECTRIC_SOURCE_ID); - originUrl.searchParams.set("source_secret", ELECTRIC_SOURCE_SECRET); - } else if (ELECTRIC_URL && ELECTRIC_SECRET) { - originUrl = new URL(ELECTRIC_URL); - originUrl.searchParams.set("secret", ELECTRIC_SECRET); + const useCloud = + request.headers.get("x-electric-backend") === "cloud" && + env.ELECTRIC_SOURCE_ID && + env.ELECTRIC_SOURCE_SECRET; + + const originUrl = useCloud + ? new URL("/v1/shape", "https://api.electric-sql.cloud") + : new URL(env.ELECTRIC_URL); + + if (useCloud) { + // biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above + originUrl.searchParams.set("source_id", env.ELECTRIC_SOURCE_ID!); + // biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above + originUrl.searchParams.set("source_secret", env.ELECTRIC_SOURCE_SECRET!); } else { - return new Response( - "Missing Electric config: set ELECTRIC_SOURCE_ID/SECRET or ELECTRIC_URL/SECRET", - { status: 500 }, - ); + originUrl.searchParams.set("secret", env.ELECTRIC_SECRET); } url.searchParams.forEach((value, key) => { @@ -89,7 +85,7 @@ export async function GET(request: Request): Promise { const response = await fetch(originUrl.toString()); const headers = new Headers(response.headers); - headers.append("Vary", "Authorization"); + headers.append("Vary", "Authorization, X-Electric-Backend"); if (headers.get("content-encoding")) { headers.delete("content-encoding"); diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index fb6840734e8..8446f6b1c95 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -10,10 +10,10 @@ export const env = createEnv({ server: { DATABASE_URL: z.string(), DATABASE_URL_UNPOOLED: z.string(), + ELECTRIC_URL: z.string().url(), + ELECTRIC_SECRET: z.string().min(16), ELECTRIC_SOURCE_ID: z.string().optional(), ELECTRIC_SOURCE_SECRET: z.string().optional(), - ELECTRIC_URL: z.string().url().optional(), - ELECTRIC_SECRET: z.string().optional(), BLOB_READ_WRITE_TOKEN: z.string(), GOOGLE_CLIENT_ID: z.string().min(1), GOOGLE_CLIENT_SECRET: z.string().min(1), diff --git a/apps/api/src/proxy.ts b/apps/api/src/proxy.ts index b979ea945f8..76eb3cabea1 100644 --- a/apps/api/src/proxy.ts +++ b/apps/api/src/proxy.ts @@ -24,7 +24,7 @@ function getCorsHeaders(origin: string | null) { "Access-Control-Allow-Origin": isAllowed ? origin : "", "Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS", "Access-Control-Allow-Headers": - "Content-Type, Authorization, x-trpc-source, trpc-accept, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed", + "Content-Type, Authorization, x-trpc-source, trpc-accept, X-Electric-Backend, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed", "Access-Control-Expose-Headers": [ // Electric sync headers "electric-offset", diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts index cbf0578aee4..b8023aa1fdd 100644 --- a/apps/desktop/electron.vite.config.ts +++ b/apps/desktop/electron.vite.config.ts @@ -77,7 +77,7 @@ export default defineConfig({ ), "process.env.STREAMS_URL": defineEnv( process.env.STREAMS_URL, - "https://streams.superset.sh", + "https://superset-stream.fly.dev", ), "process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT), "process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv( @@ -178,7 +178,7 @@ export default defineConfig({ ), "process.env.STREAMS_URL": defineEnv( process.env.STREAMS_URL, - "https://streams.superset.sh", + "https://superset-stream.fly.dev", ), "process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT), "process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv( diff --git a/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts b/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts index 3a6db6726c1..ace90e6fae6 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts @@ -82,6 +82,7 @@ const electricHeaders = { const token = getAuthToken(); return token ? `Bearer ${token}` : ""; }, + "X-Electric-Backend": "cloud", }; const organizationsCollection = createCollection( diff --git a/fly.toml b/fly.toml new file mode 100644 index 00000000000..c6fe73e4f56 --- /dev/null +++ b/fly.toml @@ -0,0 +1,32 @@ +app = "superset-electric" +primary_region = "iad" + +[build] +image = "electricsql/electric:1.4.3" + +[[vm]] +memory = "8192mb" +cpu_kind = "performance" +cpus = 4 + +[env] +ELECTRIC_DATABASE_USE_IPV6 = "true" +ELECTRIC_MAX_CONCURRENT_REQUESTS = '{"initial": 3000, "existing": 10000}' + +[http_service] +internal_port = 3000 +force_https = true +auto_stop_machines = "off" +auto_start_machines = true +min_machines_running = 1 + +[[http_service.checks]] +interval = "10s" +timeout = "2s" +grace_period = "20s" +method = "GET" +path = "/v1/health" + +[mounts] +source = "electric_data" +destination = "/var/lib/electric"