feat: adaptive progressive image loading for photo viewer

e2e/src/specs/web/photo-viewer.e2e-spec.ts

@@ -1,21 +1,25 @@
 import { AssetMediaResponseDto, LoginResponseDto } from '@immich/sdk';
-import { Page, expect, test } from '@playwright/test';
+import { expect, test } from '@playwright/test';
+import type { Socket } from 'socket.io-client';
 import { utils } from 'src/utils';
 
-function imageLocator(page: Page) {
-  return page.getByAltText('Image taken').locator('visible=true');
-}
 test.describe('Photo Viewer', () => {
   let admin: LoginResponseDto;
   let asset: AssetMediaResponseDto;
   let rawAsset: AssetMediaResponseDto;
+  let websocket: Socket;
 
   test.beforeAll(async () => {
     utils.initSdk();
     await utils.resetDatabase();
     admin = await utils.adminSetup();
     asset = await utils.createAsset(admin.accessToken);
     rawAsset = await utils.createAsset(admin.accessToken, { assetData: { filename: 'test.arw' } });
+    websocket = await utils.connectWebsocket(admin.accessToken);
+  });
+
+  test.afterAll(() => {
+    utils.disconnectWebsocket(websocket);
   });
 
   test.beforeEach(async ({ context, page }) => {
@@ -26,31 +30,51 @@ test.describe('Photo Viewer', () => {
 
   test('loads original photo when zoomed', async ({ page }) => {
     await page.goto(`/photos/${asset.id}`);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('thumbnail');
-    const box = await imageLocator(page).boundingBox();
-    expect(box).toBeTruthy();
-    const { x, y, width, height } = box!;
-    await page.mouse.move(x + width / 2, y + height / 2);
+
+    const preview = page.getByTestId('preview').filter({ visible: true });
+    await expect(preview).toHaveAttribute('src', /.+/);
+
+    const originalResponse = page.waitForResponse((response) => response.url().includes('/original'));
+
+    const { width, height } = page.viewportSize()!;
+    await page.mouse.move(width / 2, height / 2);
     await page.mouse.wheel(0, -1);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('original');
+
+    await originalResponse;
+
+    const original = page.getByTestId('original').filter({ visible: true });
+    await expect(original).toHaveAttribute('src', /original/);
   });
 
   test('loads fullsize image when zoomed and original is web-incompatible', async ({ page }) => {
     await page.goto(`/photos/${rawAsset.id}`);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('thumbnail');
-    const box = await imageLocator(page).boundingBox();
-    expect(box).toBeTruthy();
-    const { x, y, width, height } = box!;
-    await page.mouse.move(x + width / 2, y + height / 2);
+
+    const preview = page.getByTestId('preview').filter({ visible: true });
+    await expect(preview).toHaveAttribute('src', /.+/);
+
+    const fullsizeResponse = page.waitForResponse((response) => response.url().includes('fullsize'));
+
+    const { width, height } = page.viewportSize()!;
+    await page.mouse.move(width / 2, height / 2);
     await page.mouse.wheel(0, -1);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('fullsize');
+
+    await fullsizeResponse;
+
+    const original = page.getByTestId('original').filter({ visible: true });
+    await expect(original).toHaveAttribute('src', /fullsize/);
   });
 
   test('reloads photo when checksum changes', async ({ page }) => {
     await page.goto(`/photos/${asset.id}`);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('thumbnail');
-    const initialSrc = await imageLocator(page).getAttribute('src');
+
+    const preview = page.getByTestId('preview').filter({ visible: true });
+    await expect(preview).toHaveAttribute('src', /.+/);
+    const initialSrc = await preview.getAttribute('src');
+
+    const websocketEvent = utils.waitForWebsocketEvent({ event: 'assetUpdate', id: asset.id });
     await utils.replaceAsset(admin.accessToken, asset.id);
-    await expect.poll(async () => await imageLocator(page).getAttribute('src')).not.toBe(initialSrc);
+    await websocketEvent;
+
+    await expect(preview).not.toHaveAttribute('src', initialSrc!);
   });
 });

e2e/src/ui/specs/asset-viewer/broken-asset.e2e-spec.ts

@@ -64,7 +64,9 @@ test.describe('broken-asset responsiveness', () => {
 
   test('broken asset in main viewer shows icon and uses text-base', async ({ context, page }) => {
     await context.route(
-      (url) => url.pathname.includes(`/api/assets/${fixture.primaryAsset.id}/thumbnail`),
+      (url) =>
+        url.pathname.includes(`/api/assets/${fixture.primaryAsset.id}/thumbnail`) ||
+        url.pathname.includes(`/api/assets/${fixture.primaryAsset.id}/original`),
       async (route) => {
         return route.fulfill({ status: 404 });
       },
@@ -73,7 +75,7 @@ test.describe('broken-asset responsiveness', () => {
     await page.goto(`/photos/${fixture.primaryAsset.id}`);
     await page.waitForSelector('#immich-asset-viewer');
 
-    const viewerBrokenAsset = page.locator('#immich-asset-viewer #broken-asset [data-broken-asset]');
+    const viewerBrokenAsset = page.locator('[data-viewer-content] [data-broken-asset]').first();
     await expect(viewerBrokenAsset).toBeVisible();
 
     await expect(viewerBrokenAsset.locator('svg')).toBeVisible();

web/src/lib/actions/image-loader.svelte.ts

@@ -0,0 +1,25 @@
+import { cancelImageUrl } from '$lib/utils/sw-messaging';
+
+export function loadImage(src: string, onLoad: () => void, onError: () => void, onStart?: () => void) {
+  let destroyed = false;
+
+  const handleLoad = () => !destroyed && onLoad();
+  const handleError = () => !destroyed && onError();
+
+  const img = document.createElement('img');
+  img.addEventListener('load', handleLoad);
+  img.addEventListener('error', handleError);
+
+  onStart?.();
+  img.src = src;
+
+  return () => {
+    destroyed = true;
+    img.removeEventListener('load', handleLoad);
+    img.removeEventListener('error', handleError);
+    cancelImageUrl(src);
+    img.remove();
+  };
+}
+
+export type LoadImageFunction = typeof loadImage;

web/src/lib/actions/zoom-image.ts

@@ -2,7 +2,11 @@ import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
 import { createZoomImageWheel } from '@zoom-image/core';
 
 export const zoomImageAction = (node: HTMLElement, options?: { disabled?: boolean }) => {
-  const zoomInstance = createZoomImageWheel(node, { maxZoom: 10, initialState: assetViewerManager.zoomState });
+  const zoomInstance = createZoomImageWheel(node, {
+    maxZoom: 10,
+    initialState: assetViewerManager.zoomState,
+    zoomTarget: null,
+  });
 
   const unsubscribes = [
     assetViewerManager.on({ ZoomChange: (state) => zoomInstance.setState(state) }),

web/src/lib/components/AdaptiveImage.svelte

@@ -0,0 +1,228 @@
+<script lang="ts">
+  import { thumbhash } from '$lib/actions/thumbhash';
+  import AlphaBackground from '$lib/components/AlphaBackground.svelte';
+  import BrokenAsset from '$lib/components/assets/broken-asset.svelte';
+  import DelayedLoadingSpinner from '$lib/components/DelayedLoadingSpinner.svelte';
+  import ImageLayer from '$lib/components/ImageLayer.svelte';
+  import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+  import { getAssetUrls } from '$lib/utils';
+  import { AdaptiveImageLoader, type QualityList } from '$lib/utils/adaptive-image-loader.svelte';
+  import { scaleToCover, scaleToFit } from '$lib/utils/container-utils';
+  import { getAltText } from '$lib/utils/thumbnail-util';
+  import { toTimelineAsset } from '$lib/utils/timeline-util';
+  import type { AssetResponseDto, SharedLinkResponseDto } from '@immich/sdk';
+  import { untrack, type Snippet } from 'svelte';
+
+  type Props = {
+    asset: AssetResponseDto;
+    sharedLink?: SharedLinkResponseDto;
+    objectFit?: 'contain' | 'cover';
+    container: {
+      width: number;
+      height: number;
+    };
+    onUrlChange?: (url: string) => void;
+    onImageReady?: () => void;
+    onError?: () => void;
+    ref?: HTMLDivElement;
+    imgRef?: HTMLImageElement;
+    backdrop?: Snippet;
+    overlays?: Snippet;
+  };
+
+  let {
+    ref = $bindable(),
+    // eslint-disable-next-line no-useless-assignment
+    imgRef = $bindable(),
+    asset,
+    sharedLink,
+    objectFit = 'contain',
+    container,
+    onUrlChange,
+    onImageReady,
+    onError,
+    backdrop,
+    overlays,
+  }: Props = $props();
+
+  const afterThumbnail = (loader: AdaptiveImageLoader) => {
+    if (assetViewerManager.zoom > 1) {
+      loader.trigger('original');
+    } else {
+      loader.trigger('preview');
+    }
+  };
+
+  const buildQualityList = () => {
+    const assetUrls = getAssetUrls(asset, sharedLink);
+    const qualityList: QualityList = [
+      {
+        quality: 'thumbnail',
+        url: assetUrls.thumbnail,
+        onAfterLoad: afterThumbnail,
+        onAfterError: afterThumbnail,
+      },
+      {
+        quality: 'preview',
+        url: assetUrls.preview,
+        onAfterError: (loader) => loader.trigger('original'),
+      },
+      { quality: 'original', url: assetUrls.original },
+    ];
+    return qualityList;
+  };
+
+  const loaderKey = $derived(`${asset.id}:${asset.thumbhash}:${sharedLink?.id}`);
+
+  const adaptiveImageLoader = $derived.by(() => {
+    void loaderKey;
+
+    return untrack(
+      () =>
+        new AdaptiveImageLoader(buildQualityList(), {
+          onImageReady,
+          onError,
+          onUrlChange,
+        }),
+    );
+  });
+
+  $effect.pre(() => {
+    const loader = adaptiveImageLoader;
+    untrack(() => assetViewerManager.resetZoomState());
+    return () => loader.destroy();
+  });
+
+  const imageDimensions = $derived.by(() => {
+    const { width, height } = asset;
+    if (width && width > 0 && height && height > 0) {
+      return { width, height };
+    }
+    return { width: 1, height: 1 };
+  });
+
+  const { width, height, left, top } = $derived.by(() => {
+    const scaleFn = objectFit === 'cover' ? scaleToCover : scaleToFit;
+    const { width, height } = scaleFn(imageDimensions, container);
+    return {
+      width: width + 'px',
+      height: height + 'px',
+      left: (container.width - width) / 2 + 'px',
+      top: (container.height - height) / 2 + 'px',
+    };
+  });
+
+  const { status } = $derived(adaptiveImageLoader);
+  const alt = $derived(status.urls.preview ? $getAltText(toTimelineAsset(asset)) : '');
+
+  const show = $derived.by(() => {
+    const { quality, started, hasError, urls } = status;
+    return {
+      alphaBackground: !hasError && started,
+      spinner: !asset.thumbhash && !started,
+      brokenAsset: hasError,
+      thumbhash: quality.thumbnail !== 'success' && quality.preview !== 'success' && quality.original !== 'success',
+      thumbnail: quality.thumbnail !== 'error' && quality.preview !== 'success' && quality.original !== 'success',
+      preview: quality.preview !== 'error' && quality.original !== 'success',
+      original: quality.original !== 'error' && urls.original !== undefined,
+    };
+  });
+
+  $effect(() => {
+    assetViewerManager.imageLoaderStatus = status;
+  });
+
+  $effect(() => {
+    if (assetViewerManager.zoom > 1 && status.quality.original !== 'success') {
+      untrack(() => void adaptiveImageLoader.trigger('original'));
+    }
+  });
+
+  let thumbnailElement = $state<HTMLImageElement>();
+  let previewElement = $state<HTMLImageElement>();
+  let originalElement = $state<HTMLImageElement>();
+
+  $effect(() => {
+    const quality = status.quality;
+    imgRef =
+      (quality.original === 'success' ? originalElement : undefined) ??
+      (quality.preview === 'success' ? previewElement : undefined) ??
+      (quality.thumbnail === 'success' ? thumbnailElement : undefined);
+  });
+
+  const zoomTransform = $derived.by(() => {
+    const { currentZoom, currentPositionX, currentPositionY } = assetViewerManager.zoomState;
+    if (currentZoom === 1 && currentPositionX === 0 && currentPositionY === 0) {
+      return undefined;
+    }
+    return `translate(${currentPositionX}px, ${currentPositionY}px) scale(${currentZoom})`;
+  });
+</script>
+
+<div class="relative h-full w-full overflow-hidden will-change-transform" bind:this={ref}>
+  {@render backdrop?.()}
+
+  <div
+    class="absolute inset-0"
+    style:transform={zoomTransform}
+    style:transform-origin={zoomTransform ? '0 0' : undefined}
+  >
+    <div class="absolute" style:left style:top style:width style:height>
+      {#if show.alphaBackground}
+        <AlphaBackground />
+      {/if}
+
+      {#if show.thumbhash}
+        {#if asset.thumbhash}
+          <!-- Thumbhash / spinner layer  -->
+          <canvas use:thumbhash={{ base64ThumbHash: asset.thumbhash }} class="h-full w-full absolute"></canvas>
+        {:else if show.spinner}
+          <DelayedLoadingSpinner />
+        {/if}
+      {/if}
+
+      {#if show.thumbnail}
+        <ImageLayer
+          {adaptiveImageLoader}
+          {width}
+          {height}
+          quality="thumbnail"
+          src={status.urls.thumbnail}
+          alt=""
+          role="presentation"
+          bind:ref={thumbnailElement}
+        />
+      {/if}
+
+      {#if show.brokenAsset}
+        <BrokenAsset class="text-xl h-full w-full absolute" />
+      {/if}
+
+      {#if show.preview}
+        <ImageLayer
+          {adaptiveImageLoader}
+          {alt}
+          {width}
+          {height}
+          {overlays}
+          quality="preview"
+          src={status.urls.preview}
+          bind:ref={previewElement}
+        />
+      {/if}
+
+      {#if show.original}
+        <ImageLayer
+          {adaptiveImageLoader}
+          {alt}
+          {width}
+          {height}
+          {overlays}
+          quality="original"
+          src={status.urls.original}
+          bind:ref={originalElement}
+        />
+      {/if}
+    </div>
+  </div>
+</div>

web/src/lib/components/AlphaBackground.svelte

@@ -0,0 +1,11 @@
+<script lang="ts">
+  import type { ClassValue } from 'svelte/elements';
+
+  interface Props {
+    class?: ClassValue;
+  }
+
+  let { class: className = '' }: Props = $props();
+</script>
+
+<div class="absolute h-full w-full bg-gray-300 dark:bg-gray-700 {className}"></div>