diff --git a/.github/templates/preview-comment.md b/.github/templates/preview-comment.md index a1bd37c3d40..f555706ecd5 100644 --- a/.github/templates/preview-comment.md +++ b/.github/templates/preview-comment.md @@ -14,11 +14,6 @@ $DATABASE_LINK -Fly.io Electric (Fly.io) -$ELECTRIC_STATUS -$ELECTRIC_LINK - - Vercel API (Vercel) $API_STATUS $API_LINK diff --git a/.github/workflows/cleanup-preview.yml b/.github/workflows/cleanup-preview.yml index e7b4e732e26..09a6710df57 100644 --- a/.github/workflows/cleanup-preview.yml +++ b/.github/workflows/cleanup-preview.yml @@ -22,17 +22,6 @@ jobs: branch: ${{ github.event.pull_request.head.ref }} api_key: ${{ secrets.NEON_API_KEY }} - - name: Setup Fly CLI - uses: superfly/flyctl-actions/setup-flyctl@master - - - name: Delete Electric Fly.io app - id: electric-cleanup - continue-on-error: true - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: | - flyctl apps destroy "superset-electric-pr-${{ github.event.pull_request.number }}" --yes - - name: Update comment if: always() uses: thollander/actions-comment-pull-request@v3 @@ -42,7 +31,6 @@ jobs: The following preview resources have been cleaned up: - ${{ steps.neon-cleanup.outcome == 'success' && '✅' || '⚠️' }} Neon database branch - - ${{ steps.electric-cleanup.outcome == 'success' && '✅' || '⚠️' }} Electric Fly.io app Thank you for your contribution! 🎉 comment-tag: "🚀-preview-deployment" diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index 6f9bf4cfed7..749a58d7bc1 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -17,7 +17,6 @@ env: MARKETING_ALIAS: marketing-pr-${{ github.event.pull_request.number }}-superset.vercel.app ADMIN_ALIAS: admin-pr-${{ github.event.pull_request.number }}-superset.vercel.app DOCS_ALIAS: docs-pr-${{ github.event.pull_request.number }}-superset.vercel.app - ELECTRIC_URL: https://superset-electric-pr-${{ github.event.pull_request.number }}.fly.dev/v1/shape jobs: deploy-database: @@ -75,51 +74,6 @@ jobs: name: database-status path: database-status.env - deploy-electric: - name: Deploy Electric (Fly.io) - runs-on: ubuntu-latest - needs: deploy-database - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download database info - uses: actions/download-artifact@v4 - with: - name: database-status - - - name: Load database URL - run: | - source database-status.env - echo "DATABASE_URL_UNPOOLED=$DATABASE_URL_UNPOOLED" >> $GITHUB_ENV - - - name: Deploy Electric to Fly.io - uses: superfly/fly-pr-review-apps@1.3.0 - with: - name: superset-electric-pr-${{ github.event.pull_request.number }} - region: iad - org: ${{ vars.FLY_ORG }} - config: fly.toml - secrets: | - DATABASE_URL=${{ env.DATABASE_URL_UNPOOLED }} - ELECTRIC_SECRET=${{ secrets.ELECTRIC_SECRET_PREVIEW }} - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - - - name: Save Electric status - run: | - cat > electric-status.env << EOF - ELECTRIC_STATUS="✅" - ELECTRIC_LINK="View App" - EOF - - - name: Upload Electric status - uses: actions/upload-artifact@v4 - with: - name: electric-status - path: electric-status.env - deploy-api: name: Deploy API runs-on: ubuntu-latest @@ -199,8 +153,8 @@ jobs: QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }} QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }} QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }} - ELECTRIC_URL: ${{ env.ELECTRIC_URL }} - ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET_PREVIEW }} + ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }} + ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }} DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }} DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }} STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} @@ -248,8 +202,8 @@ jobs: --env QSTASH_TOKEN=$QSTASH_TOKEN \ --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \ --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \ - --env ELECTRIC_URL=$ELECTRIC_URL \ - --env ELECTRIC_SECRET=$ELECTRIC_SECRET \ + --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \ + --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \ --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \ --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \ --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \ @@ -671,7 +625,7 @@ jobs: name: Post Deployment Comment runs-on: ubuntu-latest if: always() - needs: [deploy-database, deploy-electric, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs] + needs: [deploy-database, deploy-api, deploy-web, deploy-marketing, deploy-admin, deploy-docs] permissions: contents: read pull-requests: write @@ -690,8 +644,6 @@ jobs: run: | DATABASE_STATUS="❌" DATABASE_LINK="Failed to create" - ELECTRIC_STATUS="❌" - ELECTRIC_LINK="Failed to deploy" API_STATUS="❌" API_LINK="Failed to deploy" WEB_STATUS="❌" @@ -707,10 +659,6 @@ jobs: source database-status.env fi - if [[ "${{ needs.deploy-electric.result }}" == "success" ]]; then - source electric-status.env - fi - if [[ "${{ needs.deploy-api.result }}" == "success" ]]; then source api-status.env fi @@ -731,7 +679,7 @@ jobs: source docs-status.env fi - export DATABASE_STATUS DATABASE_LINK ELECTRIC_STATUS ELECTRIC_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK + export DATABASE_STATUS DATABASE_LINK API_STATUS API_LINK WEB_STATUS WEB_LINK MARKETING_STATUS MARKETING_LINK ADMIN_STATUS ADMIN_LINK DOCS_STATUS DOCS_LINK envsubst < .github/templates/preview-comment.md > final-comment.md - name: Post final deployment comment diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 55f2a233493..7f851aad7f9 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -103,8 +103,8 @@ jobs: QSTASH_TOKEN: ${{ secrets.QSTASH_TOKEN }} QSTASH_CURRENT_SIGNING_KEY: ${{ secrets.QSTASH_CURRENT_SIGNING_KEY }} QSTASH_NEXT_SIGNING_KEY: ${{ secrets.QSTASH_NEXT_SIGNING_KEY }} - ELECTRIC_URL: ${{ secrets.ELECTRIC_URL }} - ELECTRIC_SECRET: ${{ secrets.ELECTRIC_SECRET }} + ELECTRIC_SOURCE_ID: ${{ secrets.ELECTRIC_SOURCE_ID }} + ELECTRIC_SOURCE_SECRET: ${{ secrets.ELECTRIC_SOURCE_SECRET }} DURABLE_STREAMS_URL: ${{ secrets.DURABLE_STREAMS_URL }} DURABLE_STREAMS_SECRET: ${{ secrets.DURABLE_STREAMS_SECRET }} STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} @@ -152,8 +152,8 @@ jobs: --env QSTASH_TOKEN=$QSTASH_TOKEN \ --env QSTASH_CURRENT_SIGNING_KEY=$QSTASH_CURRENT_SIGNING_KEY \ --env QSTASH_NEXT_SIGNING_KEY=$QSTASH_NEXT_SIGNING_KEY \ - --env ELECTRIC_URL=$ELECTRIC_URL \ - --env ELECTRIC_SECRET=$ELECTRIC_SECRET \ + --env ELECTRIC_SOURCE_ID=$ELECTRIC_SOURCE_ID \ + --env ELECTRIC_SOURCE_SECRET=$ELECTRIC_SOURCE_SECRET \ --env DURABLE_STREAMS_URL=$DURABLE_STREAMS_URL \ --env DURABLE_STREAMS_SECRET=$DURABLE_STREAMS_SECRET \ --env STRIPE_SECRET_KEY=$STRIPE_SECRET_KEY \ @@ -418,33 +418,6 @@ jobs: --env SECRETS_ENCRYPTION_KEY=$SECRETS_ENCRYPTION_KEY \ --env ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY - deploy-electric: - name: Deploy Electric to Fly.io - runs-on: ubuntu-latest - environment: production - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Fly CLI - uses: superfly/flyctl-actions/setup-flyctl@master - - - name: Stage secrets - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: | - flyctl secrets set \ - DATABASE_URL="${{ secrets.DATABASE_URL_UNPOOLED }}" \ - ELECTRIC_SECRET="${{ secrets.ELECTRIC_SECRET }}" \ - --app superset-electric \ - --stage - - - name: Deploy to Fly.io - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: flyctl deploy . --config fly.toml --remote-only - deploy-docs: name: Deploy Docs to Vercel runs-on: ubuntu-latest diff --git a/apps/api/src/app/api/electric/[...path]/route.ts b/apps/api/src/app/api/electric/[...path]/route.ts index 007fc8359ab..d434a2b6741 100644 --- a/apps/api/src/app/api/electric/[...path]/route.ts +++ b/apps/api/src/app/api/electric/[...path]/route.ts @@ -24,22 +24,26 @@ export async function GET(request: Request): Promise { return new Response("Not a member of this organization", { status: 403 }); } - const useCloud = - request.headers.get("x-electric-backend") === "cloud" && - env.ELECTRIC_SOURCE_ID && - env.ELECTRIC_SOURCE_SECRET; - - const originUrl = useCloud - ? new URL("/v1/shape", "https://api.electric-sql.cloud") - : new URL(env.ELECTRIC_URL); - - if (useCloud) { - // biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above - originUrl.searchParams.set("source_id", env.ELECTRIC_SOURCE_ID!); - // biome-ignore lint/style/noNonNullAssertion: guarded by useCloud check above - originUrl.searchParams.set("source_secret", env.ELECTRIC_SOURCE_SECRET!); + const { + ELECTRIC_SOURCE_ID, + ELECTRIC_SOURCE_SECRET, + ELECTRIC_URL, + ELECTRIC_SECRET, + } = env; + + let originUrl: URL; + if (ELECTRIC_SOURCE_ID && ELECTRIC_SOURCE_SECRET) { + originUrl = new URL("/v1/shape", "https://api.electric-sql.cloud"); + originUrl.searchParams.set("source_id", ELECTRIC_SOURCE_ID); + originUrl.searchParams.set("source_secret", ELECTRIC_SOURCE_SECRET); + } else if (ELECTRIC_URL && ELECTRIC_SECRET) { + originUrl = new URL(ELECTRIC_URL); + originUrl.searchParams.set("secret", ELECTRIC_SECRET); } else { - originUrl.searchParams.set("secret", env.ELECTRIC_SECRET); + return new Response( + "Missing Electric config: set ELECTRIC_SOURCE_ID/SECRET or ELECTRIC_URL/SECRET", + { status: 500 }, + ); } url.searchParams.forEach((value, key) => { @@ -85,7 +89,7 @@ export async function GET(request: Request): Promise { const response = await fetch(originUrl.toString()); const headers = new Headers(response.headers); - headers.append("Vary", "Authorization, X-Electric-Backend"); + headers.append("Vary", "Authorization"); if (headers.get("content-encoding")) { headers.delete("content-encoding"); diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 8446f6b1c95..fb6840734e8 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -10,10 +10,10 @@ export const env = createEnv({ server: { DATABASE_URL: z.string(), DATABASE_URL_UNPOOLED: z.string(), - ELECTRIC_URL: z.string().url(), - ELECTRIC_SECRET: z.string().min(16), ELECTRIC_SOURCE_ID: z.string().optional(), ELECTRIC_SOURCE_SECRET: z.string().optional(), + ELECTRIC_URL: z.string().url().optional(), + ELECTRIC_SECRET: z.string().optional(), BLOB_READ_WRITE_TOKEN: z.string(), GOOGLE_CLIENT_ID: z.string().min(1), GOOGLE_CLIENT_SECRET: z.string().min(1), diff --git a/apps/api/src/proxy.ts b/apps/api/src/proxy.ts index 76eb3cabea1..b979ea945f8 100644 --- a/apps/api/src/proxy.ts +++ b/apps/api/src/proxy.ts @@ -24,7 +24,7 @@ function getCorsHeaders(origin: string | null) { "Access-Control-Allow-Origin": isAllowed ? origin : "", "Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS", "Access-Control-Allow-Headers": - "Content-Type, Authorization, x-trpc-source, trpc-accept, X-Electric-Backend, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed", + "Content-Type, Authorization, x-trpc-source, trpc-accept, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Closed", "Access-Control-Expose-Headers": [ // Electric sync headers "electric-offset", diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts index b8023aa1fdd..cbf0578aee4 100644 --- a/apps/desktop/electron.vite.config.ts +++ b/apps/desktop/electron.vite.config.ts @@ -77,7 +77,7 @@ export default defineConfig({ ), "process.env.STREAMS_URL": defineEnv( process.env.STREAMS_URL, - "https://superset-stream.fly.dev", + "https://streams.superset.sh", ), "process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT), "process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv( @@ -178,7 +178,7 @@ export default defineConfig({ ), "process.env.STREAMS_URL": defineEnv( process.env.STREAMS_URL, - "https://superset-stream.fly.dev", + "https://streams.superset.sh", ), "process.env.DESKTOP_VITE_PORT": defineEnv(process.env.DESKTOP_VITE_PORT), "process.env.DESKTOP_NOTIFICATIONS_PORT": defineEnv( diff --git a/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts b/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts index ace90e6fae6..3a6db6726c1 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/providers/CollectionsProvider/collections.ts @@ -82,7 +82,6 @@ const electricHeaders = { const token = getAuthToken(); return token ? `Bearer ${token}` : ""; }, - "X-Electric-Backend": "cloud", }; const organizationsCollection = createCollection( diff --git a/fly.toml b/fly.toml deleted file mode 100644 index c6fe73e4f56..00000000000 --- a/fly.toml +++ /dev/null @@ -1,32 +0,0 @@ -app = "superset-electric" -primary_region = "iad" - -[build] -image = "electricsql/electric:1.4.3" - -[[vm]] -memory = "8192mb" -cpu_kind = "performance" -cpus = 4 - -[env] -ELECTRIC_DATABASE_USE_IPV6 = "true" -ELECTRIC_MAX_CONCURRENT_REQUESTS = '{"initial": 3000, "existing": 10000}' - -[http_service] -internal_port = 3000 -force_https = true -auto_stop_machines = "off" -auto_start_machines = true -min_machines_running = 1 - -[[http_service.checks]] -interval = "10s" -timeout = "2s" -grace_period = "20s" -method = "GET" -path = "/v1/health" - -[mounts] -source = "electric_data" -destination = "/var/lib/electric"