feat(web-console): export parquet files, fix unresponsive CSV imports

packages/browser-tests/.gitignore

@@ -1,3 +1,4 @@
 node_modules
 cypress/videos
 cypress/screenshots
+cypress/downloads

packages/browser-tests/cypress/integration/console/download.spec.js

@@ -0,0 +1,171 @@
+describe("download functionality", () => {
+  beforeEach(() => {
+    cy.loadConsoleWithAuth();
+  });
+
+  it("should show download button with results", () => {
+    // When
+    cy.typeQuery("select x from long_sequence(10)");
+    cy.runLine();
+
+    // Then
+    cy.getByDataHook("download-parquet-button").should("be.visible");
+    cy.getByDataHook("download-dropdown-button").should("be.visible");
+    cy.getByDataHook("download-csv-button").should("not.exist");
+
+    // When
+    cy.getByDataHook("download-dropdown-button").click();
+
+    // Then
+    cy.getByDataHook("download-csv-button").should("be.visible");
+  });
+
+  it("should trigger CSV download", () => {
+    const query = "select x from long_sequence(10)";
+
+    // Given
+    cy.intercept("GET", "**/exp?*", (req) => {
+      req.reply({
+        statusCode: 200,
+        body: null,
+      });
+    }).as("exportRequest");
+
+    // When
+    cy.typeQuery(query);
+    cy.runLine();
+    cy.getByDataHook("download-dropdown-button").click();
+    cy.getByDataHook("download-csv-button").click();
+
+    // Then
+    cy.wait("@exportRequest").then((interception) => {
+      expect(interception.request.url).to.include("fmt=csv");
+      expect(interception.request.url).to.include(
+        encodeURIComponent(query.replace(/\s+/g, " "))
+      );
+    });
+  });
+
+  it("should trigger Parquet download", () => {
+    const query = "select x from long_sequence(10)";
+
+    // Given
+    cy.intercept("GET", "**/exp?*", (req) => {
+      req.reply({
+        statusCode: 200,
+        body: null,
+      });
+    }).as("exportRequest");
+
+    // When
+    cy.typeQuery(query);
+    cy.runLine();
+    cy.getByDataHook("download-parquet-button").click();
+
+    // Then
+    cy.wait("@exportRequest").then((interception) => {
+      expect(interception.request.url).to.include("fmt=parquet");
+      expect(interception.request.url).to.include("parquetVersion=1");
+      expect(interception.request.url).to.include(
+        encodeURIComponent(query.replace(/\s+/g, " "))
+      );
+    });
+  });
+
+  it("should show error toast on bad request", () => {
+    // Given
+    cy.intercept("GET", "**/exp?*", (req) => {
+      const url = new URL(req.url);
+      url.searchParams.set("fmt", "badformat");
+      req.url = url.toString();
+    }).as("badExportRequest");
+
+    // When
+    cy.typeQuery("select x from long_sequence(5)");
+    cy.runLine();
+    cy.getByDataHook("download-dropdown-button").click();
+    cy.getByDataHook("download-csv-button").click();
+
+    // Then
+    cy.wait("@badExportRequest").then(() => {
+      cy.getByRole("alert").should(
+        "contain",
+        "Download failed with status code 400: unrecognised format [format=badformat]"
+      );
+    });
+  });
+
+  it("should show error toast on server error", () => {
+    // Given
+    cy.intercept("GET", "**/exp?*", (req) => {
+      req.reply({
+        statusCode: 500,
+      });
+    }).as("serverErrorRequest");
+
+    // When
+    cy.typeQuery("select x from long_sequence(5)");
+    cy.runLine();
+    cy.getByDataHook("download-dropdown-button").click();
+    cy.getByDataHook("download-csv-button").click();
+
+    // Then
+    cy.wait("@serverErrorRequest").then(() => {
+      cy.getByRole("alert").should(
+        "contain",
+        "Download failed with status code 500: Internal Server Error"
+      );
+    });
+  });
+
+  it("should show loading spinner when downloading", () => {
+    // Given
+    cy.intercept("GET", "**/exp?*", (req) => {
+      req.reply({
+        statusCode: 200,
+        body: null,
+        delay: 1000,
+      });
+    }).as("exportRequest");
+
+    // When
+    cy.typeQuery("select * from long_sequence(10)");
+    cy.runLine();
+    cy.getByDataHook("download-parquet-button").click();
+
+    // Then
+    cy.getByDataHook("download-loading-indicator").should("be.visible");
+
+    // Then
+    cy.wait("@exportRequest").then(() => {
+      cy.getByDataHook("download-loading-indicator").should("not.exist");
+    });
+  });
+
+  it("should download the file", () => {
+    const query = "select x from long_sequence(10)";
+    // Given
+    cy.intercept("GET", "**/exp?*").as("exportRequest");
+
+    // When
+    cy.typeQuery(query);
+    cy.runLine();
+    cy.getByDataHook("download-dropdown-button").click();
+    cy.getByDataHook("download-csv-button").click();
+
+    // Then
+    cy.wait("@exportRequest").then((interception) => {
+      expect(interception.request.url).to.include("fmt=csv");
+      expect(interception.request.url).to.include(
+        encodeURIComponent(query.replace(/\s+/g, " "))
+      );
+      const filename = new URL(interception.request.url).searchParams.get(
+        "filename"
+      );
+      cy.readFile(`cypress/downloads/${filename}.csv`).should(
+        "eq",
+        '"x"\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9\r\n10\r\n'
+      );
+    });
+  });
+});

packages/web-console/public/download-sw.js

@@ -0,0 +1,91 @@
+let authToken = null
+
+self.addEventListener('message', (event) => {
+  if (event.data && event.data.type === 'SET_AUTH_TOKEN') {
+    authToken = event.data.token
+    self.clients.matchAll().then((clients) => {
+      clients.forEach((client) => {
+        client.postMessage({
+          type: 'AUTH_TOKEN_ACK',
+        })
+      })
+    })
+  }
+})
+
+self.addEventListener('fetch', (event) => {
+  const url = new URL(event.request.url)
+  if (url.searchParams.get('noAuth')) {
+    return
+  }
+
+  if (url.pathname === '/exp' || url.pathname.endsWith('/exp')) {
+    const requestKey = new URL(event.request.url).searchParams.get('filename')
+
+    event.respondWith(
+      (async () => {
+        try {
+          const headers = new Headers(event.request.headers)
+          if (authToken) {
+            headers.set('Authorization', authToken)
+          }
+
+          const modifiedRequest = new Request(event.request, {
+            headers: headers,
+          })
+
+          const response = await fetch(modifiedRequest)
+
+          if (!response.ok) {
+            let message = response.statusText
+            try {
+              const json = await response.json()
+              const errorMessage = json.error
+              if (errorMessage) {
+                message = errorMessage
+              }
+            } catch (_) {}
+
+            self.clients.matchAll().then((clients) => {
+              clients.forEach((client) => {
+                client.postMessage({
+                  type: `DOWNLOAD_ERROR_${requestKey}`,
+                  status: response.status,
+                  message,
+                })
+              })
+            })
+          } else {
+            self.clients.matchAll().then((clients) => {
+              clients.forEach((client) => {
+                client.postMessage({
+                  type: `DOWNLOAD_START_${requestKey}`,
+                })
+              })
+            })
+          }
+          return response
+        } catch (error) {
+          self.clients.matchAll().then((clients) => {
+            clients.forEach((client) => {
+              client.postMessage({
+                type: `DOWNLOAD_ERROR_${requestKey}`,
+                status: 500,
+                message: error.message ?? 'Internal server error',
+              })
+            })
+          })
+          console.error('[SW] Download service worker error:', error)
+        }
+      })()
+    );
+  }
+});
+
+self.addEventListener('install', (_) => {
+  self.skipWaiting()
+});
+
+self.addEventListener('activate', (event) => {
+  event.waitUntil(self.clients.claim())
+});

packages/web-console/serve-dist.js

@@ -22,7 +22,8 @@ const server = http.createServer((req, res) => {
     reqPathName.startsWith("/settings") ||
     reqPathName.startsWith("/warnings") ||
     reqPathName.startsWith("/chk") ||
-    reqPathName.startsWith("/imp")
+    reqPathName.startsWith("/imp") ||
+    reqPathName.startsWith("/exp")
   ) {
     // proxy /exec requests to localhost:9000
     const options = {

packages/web-console/src/components/LoadingSpinner/index.tsx

@@ -0,0 +1,24 @@
+import React from "react"
+import styled, { useTheme } from "styled-components"
+import { Loader3 } from "@styled-icons/remix-line"
+import { Color } from "../../types"
+import { spinAnimation } from "../../components/Animation"
+
+const StyledLoader = styled(Loader3)<{ $size: string, $color: Color }>`
+  width: ${({ $size }) => $size};
+  height: ${({ $size }) => $size};
+  color: ${({ $color, theme }) => $color ? theme.color[$color] : theme.color.pink};
+  ${spinAnimation};
+`
+
+type Props = {
+  size?: string
+  color?: Color
+}
+
+export const LoadingSpinner = ({ size = "18px", color = "pink" }: Props) => {
+  const theme = useTheme()
+  return (
+    <StyledLoader data-hook="loading-spinner" $size={size} $color={color} theme={theme} />
+  )
+}

packages/web-console/src/components/index.ts

@@ -31,6 +31,7 @@ export * from "./Hooks"
 export * from "./IconWithTooltip"
 export * from "./Input"
 export * from "./Link"
+export * from "./LoadingSpinner"
 export * from "./PaneContent"
 export * from "./PaneMenu"
 export * from "./PaneWrapper"

packages/web-console/src/index.tsx

@@ -45,6 +45,7 @@ import Layout from "./scenes/Layout"
 import { theme } from "./theme"
 import { LocalStorageProvider } from "./providers/LocalStorageProvider"
 import { AuthProvider, QuestProvider, SettingsProvider, PosthogProviderWrapper } from "./providers"
+import { registerDownloadServiceWorker } from "./utils/serviceWorker"
 
 const epicMiddleware = createEpicMiddleware<
   StoreAction,
@@ -63,6 +64,8 @@ const FadeSlow = createGlobalFadeTransition(
   TransitionDuration.SLOW,
 )
 
+registerDownloadServiceWorker()
+
 ReactDOM.render(
   <ThemeProvider theme={theme}>
     <ScreenSizeProvider>