From ee63153d860e39373dcf61e6af47ff21ea8a3f44 Mon Sep 17 00:00:00 2001 From: Grzegorz Zdunek Date: Wed, 26 Feb 2025 18:26:24 +0100 Subject: [PATCH 1/2] Refactor client canvas (#52431) * Remove non-canvas related listeners from `TdpClientCanvas` * Simplify event handlers setup by using React handlers * Listen to canvas resize events instead of window ones * Expose imperative API to set pointer * Expose imperative API to render frames * Expose imperative API to change canvas resolution Also, we no longer need to set the canvas size manually; it is now handled by `objectFit: scale-down`. * Remove duplicate player shutdown * Expose imperative API to clear canvas * Remove unnecessary display size syncing We get the canvas resolution from the server, there doesn't seem to be a need to calculate it from the browser window size. * Fix focusing canvas It didn't work because it happened too early, before we even showed the MFA dialog which stole focus. This is a temporary fix, I will do it more elegantly in the future. * Minor code changes Remove component memoization, it didn't work anyway and doesn't seem needed. We don't have anything particularly expensive to render here from React's perspective. * Add a TODO for improving rendering functions (cherry picked from commit 671475f7de24b1b9d52307e6b70ccbf63b6209ee) --- .../DesktopSession/DesktopSession.story.tsx | 51 +- .../src/DesktopSession/DesktopSession.tsx | 223 ++++++- .../src/DesktopSession/useTdpClientCanvas.tsx | 133 ++-- .../teleport/src/Player/DesktopPlayer.tsx | 177 ++--- .../TdpClientCanvas/TdpClientCanvas.tsx | 616 ++++++------------ .../src/components/TdpClientCanvas/index.ts | 2 +- 6 files changed, 547 insertions(+), 655 deletions(-) diff --git a/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx b/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx index a8d3417302831..5579875be1f2f 100644 --- a/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx +++ b/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx @@ -20,9 +20,9 @@ import { useState } from 'react'; import { ButtonPrimary } from 'design/Button'; import { NotificationItem } from 'shared/components/Notification'; -import { throttle } from 'shared/utils/highbar'; import { TdpClient, TdpClientEvent } from 'teleport/lib/tdp'; +import { BitmapFrame } from 'teleport/lib/tdp/client'; import { makeDefaultMfaState } from 'teleport/lib/useMfa'; import { DesktopSession } from './DesktopSession'; @@ -38,12 +38,6 @@ const fakeClient = () => { return client; }; -const fillGray = (canvas: HTMLCanvasElement) => { - var ctx = canvas.getContext('2d'); - ctx.fillStyle = 'gray'; - ctx.fillRect(0, 0, canvas.width, canvas.height); -}; - const props: State = { hostname: 'host.com', fetchAttempt: { status: 'processing' }, @@ -66,9 +60,7 @@ const props: State = { setDirectorySharingState: () => {}, onShareDirectory: () => {}, onCtrlAltDel: () => {}, - clientOnPngFrame: () => {}, - clientOnBitmapFrame: () => {}, - clientOnClientScreenSpec: () => {}, + setInitialTdpConnectionSucceeded: () => {}, clientScreenSpecToRequest: { width: 0, height: 0 }, clientOnTdpError: () => {}, clientOnTdpInfo: () => {}, @@ -88,7 +80,7 @@ const props: State = { setShowAnotherSessionActiveDialog: () => {}, alerts: [], onRemoveAlert: () => {}, - windowOnResize: throttle(() => {}, 1000), + onResize: () => {}, }; export const BothProcessing = () => ( @@ -172,7 +164,7 @@ export const TdpGraceful = () => ( export const ConnectedSettingsFalse = () => { const client = fakeClient(); client.connect = async () => { - client.emit(TdpClientEvent.TDP_PNG_FRAME); + emitGrayFrame(client); }; return ( @@ -193,9 +185,6 @@ export const ConnectedSettingsFalse = () => { browserSupported: false, directorySelected: false, }} - clientOnPngFrame={(ctx: CanvasRenderingContext2D) => { - fillGray(ctx.canvas); - }} /> ); }; @@ -203,7 +192,7 @@ export const ConnectedSettingsFalse = () => { export const ConnectedSettingsTrue = () => { const client = fakeClient(); client.connect = async () => { - client.emit(TdpClientEvent.TDP_PNG_FRAME); + emitGrayFrame(client); }; return ( @@ -224,9 +213,6 @@ export const ConnectedSettingsTrue = () => { browserSupported: true, directorySelected: true, }} - clientOnPngFrame={(ctx: CanvasRenderingContext2D) => { - fillGray(ctx.canvas); - }} /> ); }; @@ -319,7 +305,7 @@ export const ClipboardSharingDisabledBrowserPermissions = () => ( export const Alerts = () => { const client = fakeClient(); client.connect = async () => { - client.emit(TdpClientEvent.TDP_PNG_FRAME); + emitGrayFrame(client); }; const [alerts, setAlerts] = useState([]); @@ -375,12 +361,31 @@ export const Alerts = () => { browserSupported: true, directorySelected: true, }} - clientOnPngFrame={(ctx: CanvasRenderingContext2D) => { - fillGray(ctx.canvas); - }} alerts={alerts} onRemoveAlert={removeAlert} /> ); }; + +function emitGrayFrame(client: TdpClient) { + const width = 300; + const height = 100; + const imageData = new ImageData(width, height); + + // Fill with gray (RGB: 128, 128, 128) + for (let i = 0; i < imageData.data.length; i += 4) { + imageData.data[i] = 128; // Red + imageData.data[i + 1] = 128; // Green + imageData.data[i + 2] = 128; // Blue + imageData.data[i + 3] = 255; // Alpha (fully opaque) + } + + const frame: BitmapFrame = { + left: 0, + top: 0, + image_data: imageData, + }; + + client.emit(TdpClientEvent.TDP_BMP_FRAME, frame); +} diff --git a/web/packages/teleport/src/DesktopSession/DesktopSession.tsx b/web/packages/teleport/src/DesktopSession/DesktopSession.tsx index f8422dd385c4c..c14c4f35a77e5 100644 --- a/web/packages/teleport/src/DesktopSession/DesktopSession.tsx +++ b/web/packages/teleport/src/DesktopSession/DesktopSession.tsx @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import { useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { Box, ButtonPrimary, ButtonSecondary, Flex, Indicator } from 'design'; import { Info } from 'design/Alert'; @@ -30,6 +30,10 @@ import { Attempt } from 'shared/hooks/useAttemptNext'; import AuthnDialog from 'teleport/components/AuthnDialog'; import TdpClientCanvas from 'teleport/components/TdpClientCanvas'; +import { TdpClientCanvasRef } from 'teleport/components/TdpClientCanvas/TdpClientCanvas'; +import { TdpClientEvent } from 'teleport/lib/tdp'; +import { BitmapFrame } from 'teleport/lib/tdp/client'; +import { ClientScreenSpec, PngFrame } from 'teleport/lib/tdp/codec'; import type { MfaState } from 'teleport/lib/useMfa'; import TopBar from './TopBar'; @@ -56,14 +60,12 @@ declare global { export function DesktopSession(props: State) { const { mfa, - tdpClient, + tdpClient: client, username, hostname, directorySharingState, setDirectorySharingState, - clientOnPngFrame, - clientOnBitmapFrame, - clientOnClientScreenSpec, + setInitialTdpConnectionSucceeded, clientOnClipboardData, clientOnTdpError, clientOnTdpWarning, @@ -78,7 +80,7 @@ export function DesktopSession(props: State) { canvasOnMouseUp, canvasOnMouseWheelScroll, canvasOnContextMenu, - windowOnResize, + onResize, clientScreenSpecToRequest, clipboardSharingState, setClipboardSharingState, @@ -97,6 +99,87 @@ export function DesktopSession(props: State) { canvasState: { shouldConnect: false, shouldDisplay: false }, }); + useEffect(() => { + if (client && clientOnClipboardData) { + client.on(TdpClientEvent.TDP_CLIPBOARD_DATA, clientOnClipboardData); + + return () => { + client.removeListener( + TdpClientEvent.TDP_CLIPBOARD_DATA, + clientOnClipboardData + ); + }; + } + }, [client, clientOnClipboardData]); + + useEffect(() => { + if (client && clientOnTdpError) { + client.on(TdpClientEvent.TDP_ERROR, clientOnTdpError); + client.on(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); + + return () => { + client.removeListener(TdpClientEvent.TDP_ERROR, clientOnTdpError); + client.removeListener(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); + }; + } + }, [client, clientOnTdpError]); + + useEffect(() => { + if (client && clientOnTdpWarning) { + client.on(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); + client.on(TdpClientEvent.CLIENT_WARNING, clientOnTdpWarning); + + return () => { + client.removeListener(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); + client.removeListener( + TdpClientEvent.CLIENT_WARNING, + clientOnTdpWarning + ); + }; + } + }, [client, clientOnTdpWarning]); + + useEffect(() => { + if (client && clientOnTdpInfo) { + client.on(TdpClientEvent.TDP_INFO, clientOnTdpInfo); + + return () => { + client.removeListener(TdpClientEvent.TDP_INFO, clientOnTdpInfo); + }; + } + }, [client, clientOnTdpInfo]); + + useEffect(() => { + if (client && clientOnWsClose) { + client.on(TdpClientEvent.WS_CLOSE, clientOnWsClose); + + return () => { + client.removeListener(TdpClientEvent.WS_CLOSE, clientOnWsClose); + }; + } + }, [client, clientOnWsClose]); + + useEffect(() => { + if (client && clientOnWsOpen) { + client.on(TdpClientEvent.WS_OPEN, clientOnWsOpen); + + return () => { + client.removeListener(TdpClientEvent.WS_OPEN, clientOnWsOpen); + }; + } + }, [client, clientOnWsOpen]); + + const { shouldConnect } = screenState.canvasState; + // Call connect after all listeners have been registered + useEffect(() => { + if (client && shouldConnect) { + client.connect(clientScreenSpecToRequest); + return () => { + client.shutdown(); + }; + } + }, [client, shouldConnect]); + // Calculate the next `ScreenState` whenever any of the constituent pieces of state change. useEffect(() => { setScreenState(prevState => @@ -117,8 +200,100 @@ export function DesktopSession(props: State) { mfa, ]); + const tdpClientCanvasRef = useRef(null); + + useEffect(() => { + if (!client) { + return; + } + const setPointer = tdpClientCanvasRef.current?.setPointer; + client.addListener(TdpClientEvent.POINTER, setPointer); + + return () => { + client.removeListener(TdpClientEvent.POINTER, setPointer); + }; + }, [client]); + + const onInitialTdpConnectionSucceeded = useCallback(() => { + setInitialTdpConnectionSucceeded(() => { + // TODO(gzdunek): This callback is a temporary fix for focusing the canvas. + // Focus the canvas once we start rendering frames. + // The timeout it a small hack, we should verify + // what is the earliest moment we can focus the canvas. + setTimeout(() => { + tdpClientCanvasRef.current?.focus(); + }, 100); + }); + }, [setInitialTdpConnectionSucceeded]); + + useEffect(() => { + if (!client) { + return; + } + const renderFrame = (frame: PngFrame) => { + onInitialTdpConnectionSucceeded(); + tdpClientCanvasRef.current?.renderPngFrame(frame); + }; + client.addListener(TdpClientEvent.TDP_PNG_FRAME, renderFrame); + + return () => { + client.removeListener(TdpClientEvent.TDP_PNG_FRAME, renderFrame); + }; + }, [client, onInitialTdpConnectionSucceeded]); + + useEffect(() => { + if (!client) { + return; + } + const renderFrame = (frame: BitmapFrame) => { + onInitialTdpConnectionSucceeded(); + tdpClientCanvasRef.current?.renderBitmapFrame(frame); + }; + client.addListener(TdpClientEvent.TDP_BMP_FRAME, renderFrame); + + return () => { + client.removeListener(TdpClientEvent.TDP_BMP_FRAME, renderFrame); + }; + }, [client, onInitialTdpConnectionSucceeded]); + + useEffect(() => { + if (!client) { + return; + } + const clear = () => tdpClientCanvasRef.current?.clear(); + client.addListener(TdpClientEvent.RESET, clear); + + return () => { + client.removeListener(TdpClientEvent.RESET, clear); + }; + }, [client]); + + useEffect(() => { + if (!client) { + return; + } + const setResolution = (spec: ClientScreenSpec) => + tdpClientCanvasRef.current?.setResolution(spec); + client.addListener(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, setResolution); + + return () => { + client.removeListener( + TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, + setResolution + ); + }; + }, [client]); + return ( - + { setClipboardSharingState(prevState => ({ @@ -129,7 +304,7 @@ export function DesktopSession(props: State) { ...prevState, isSharing: false, })); - tdpClient.shutdown(); + client.shutdown(); }} userHost={`${username}@${hostname}`} canShareDirectory={directorySharingPossible(directorySharingState)} @@ -152,31 +327,19 @@ export function DesktopSession(props: State) { {screenState.screen === 'processing' && } ); diff --git a/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx b/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx index 28b5f0e3038ab..3a7c5fe987270 100644 --- a/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx +++ b/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx @@ -17,20 +17,14 @@ * along with this program. If not, see . */ -import { useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { NotificationItem } from 'shared/components/Notification'; import { Attempt } from 'shared/hooks/useAttemptNext'; -import { debounce } from 'shared/utils/highbar'; import cfg from 'teleport/config'; import { ButtonState, ScrollAxis, TdpClient } from 'teleport/lib/tdp'; -import type { BitmapFrame } from 'teleport/lib/tdp/client'; -import { - ClientScreenSpec, - ClipboardData, - PngFrame, -} from 'teleport/lib/tdp/codec'; +import { ClipboardData } from 'teleport/lib/tdp/codec'; import { Sha256Digest } from 'teleport/lib/util'; import { getHostName } from 'teleport/services/api'; @@ -89,57 +83,17 @@ export default function useTdpClientCanvas(props: Props) { setTdpClient(new TdpClient(addr)); }, [clusterId, username, desktopName]); - /** - * Synchronize the canvas resolution and display size with the - * given ClientScreenSpec. - */ - const syncCanvas = (canvas: HTMLCanvasElement, spec: ClientScreenSpec) => { - const { width, height } = spec; - canvas.width = width; - canvas.height = height; - console.debug(`set canvas.width x canvas.height to ${width} x ${height}`); - canvas.style.width = `${width}px`; - canvas.style.height = `${height}px`; - console.debug( - `set canvas.style.width x canvas.style.height to ${width} x ${height}` - ); - }; - - // Default TdpClientEvent.TDP_PNG_FRAME handler (buffered) - const clientOnPngFrame = ( - ctx: CanvasRenderingContext2D, - pngFrame: PngFrame - ) => { - // The first image fragment we see signals a successful TDP connection. - if (!initialTdpConnectionSucceeded.current) { - syncCanvas(ctx.canvas, getDisplaySize()); - setTdpConnection({ status: 'success' }); - initialTdpConnectionSucceeded.current = true; - } - ctx.drawImage(pngFrame.data, pngFrame.left, pngFrame.top); - }; - - // Default TdpClientEvent.TDP_BMP_FRAME handler (buffered) - const clientOnBitmapFrame = ( - ctx: CanvasRenderingContext2D, - bmpFrame: BitmapFrame - ) => { - // The first image fragment we see signals a successful TDP connection. - if (!initialTdpConnectionSucceeded.current) { - setTdpConnection({ status: 'success' }); - initialTdpConnectionSucceeded.current = true; - } - ctx.putImageData(bmpFrame.image_data, bmpFrame.left, bmpFrame.top); - }; - - // Default TdpClientEvent.TDP_CLIENT_SCREEN_SPEC handler. - const clientOnClientScreenSpec = ( - cli: TdpClient, - canvas: HTMLCanvasElement, - spec: ClientScreenSpec - ) => { - syncCanvas(canvas, spec); - }; + const setInitialTdpConnectionSucceeded = useCallback( + (callback: () => void) => { + // The first image fragment we see signals a successful TDP connection. + if (!initialTdpConnectionSucceeded.current) { + callback(); + setTdpConnection({ status: 'success' }); + initialTdpConnectionSucceeded.current = true; + } + }, + [setTdpConnection] + ); // Default TdpClientEvent.TDP_CLIPBOARD_DATA handler. const clientOnClipboardData = async (clipboardData: ClipboardData) => { @@ -205,10 +159,10 @@ export default function useTdpClientCanvas(props: Props) { setWsConnection({ status: 'open' }); }; - const canvasOnKeyDown = (cli: TdpClient, e: KeyboardEvent) => { + const canvasOnKeyDown = (e: React.KeyboardEvent) => { keyboardHandler.current.handleKeyboardEvent({ - cli, - e, + cli: tdpClient, + e: e.nativeEvent, state: ButtonState.DOWN, }); @@ -220,14 +174,14 @@ export default function useTdpClientCanvas(props: Props) { // Opportunistically sync local clipboard to remote while // transient user activation is in effect. // https://developer.mozilla.org/en-US/docs/Web/API/Clipboard/readText#security - sendLocalClipboardToRemote(cli); + sendLocalClipboardToRemote(tdpClient); } }; - const canvasOnKeyUp = (cli: TdpClient, e: KeyboardEvent) => { + const canvasOnKeyUp = (e: React.KeyboardEvent) => { keyboardHandler.current.handleKeyboardEvent({ - cli, - e, + cli: tdpClient, + e: e.nativeEvent, state: ButtonState.UP, }); }; @@ -236,60 +190,53 @@ export default function useTdpClientCanvas(props: Props) { keyboardHandler.current.onFocusOut(); }; - const canvasOnMouseMove = ( - cli: TdpClient, - canvas: HTMLCanvasElement, - e: MouseEvent - ) => { - const rect = canvas.getBoundingClientRect(); + const canvasOnMouseMove = (e: React.MouseEvent) => { + const rect = e.currentTarget.getBoundingClientRect(); const x = e.clientX - rect.left; const y = e.clientY - rect.top; - cli.sendMouseMove(x, y); + tdpClient.sendMouseMove(x, y); }; - const canvasOnMouseDown = (cli: TdpClient, e: MouseEvent) => { + const canvasOnMouseDown = (e: React.MouseEvent) => { if (e.button === 0 || e.button === 1 || e.button === 2) { - cli.sendMouseButton(e.button, ButtonState.DOWN); + tdpClient.sendMouseButton(e.button, ButtonState.DOWN); } // Opportunistically sync local clipboard to remote while // transient user activation is in effect. // https://developer.mozilla.org/en-US/docs/Web/API/Clipboard/readText#security - sendLocalClipboardToRemote(cli); + sendLocalClipboardToRemote(tdpClient); }; - const canvasOnMouseUp = (cli: TdpClient, e: MouseEvent) => { + const canvasOnMouseUp = (e: React.MouseEvent) => { if (e.button === 0 || e.button === 1 || e.button === 2) { - cli.sendMouseButton(e.button, ButtonState.UP); + tdpClient.sendMouseButton(e.button, ButtonState.UP); } }; - const canvasOnMouseWheelScroll = (cli: TdpClient, e: WheelEvent) => { + const canvasOnMouseWheelScroll = (e: WheelEvent) => { e.preventDefault(); // We only support pixel scroll events, not line or page events. // https://developer.mozilla.org/en-US/docs/Web/API/WheelEvent/deltaMode if (e.deltaMode === WheelEvent.DOM_DELTA_PIXEL) { if (e.deltaX) { - cli.sendMouseWheelScroll(ScrollAxis.HORIZONTAL, -e.deltaX); + tdpClient.sendMouseWheelScroll(ScrollAxis.HORIZONTAL, -e.deltaX); } if (e.deltaY) { - cli.sendMouseWheelScroll(ScrollAxis.VERTICAL, -e.deltaY); + tdpClient.sendMouseWheelScroll(ScrollAxis.VERTICAL, -e.deltaY); } } }; // Block browser context menu so as not to obscure the context menu // on the remote machine. - const canvasOnContextMenu = () => false; + const canvasOnContextMenu = (e: React.MouseEvent) => { + e.preventDefault(); + }; - const windowOnResize = debounce( - (cli: TdpClient) => { - const spec = getDisplaySize(); - cli.resize(spec); - }, - 250, - { trailing: true } - ); + const onResize = (e: { height: number; width: number }) => { + tdpClient.resize(e); + }; const sendLocalClipboardToRemote = async (cli: TdpClient) => { if (await sysClipboardGuard(clipboardSharingState, 'read')) { @@ -309,9 +256,7 @@ export default function useTdpClientCanvas(props: Props) { return { tdpClient, clientScreenSpecToRequest: getDisplaySize(), - clientOnPngFrame, - clientOnBitmapFrame, - clientOnClientScreenSpec, + setInitialTdpConnectionSucceeded, clientOnTdpError, clientOnClipboardData, clientOnWsClose, @@ -326,7 +271,7 @@ export default function useTdpClientCanvas(props: Props) { canvasOnMouseUp, canvasOnMouseWheelScroll, canvasOnContextMenu, - windowOnResize, + onResize, }; } diff --git a/web/packages/teleport/src/Player/DesktopPlayer.tsx b/web/packages/teleport/src/Player/DesktopPlayer.tsx index b26db1f8a1242..dcfbeec0b98d4 100644 --- a/web/packages/teleport/src/Player/DesktopPlayer.tsx +++ b/web/packages/teleport/src/Player/DesktopPlayer.tsx @@ -16,31 +16,23 @@ * along with this program. If not, see . */ -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import styled from 'styled-components'; import { Alert, Box, Flex, Indicator } from 'design'; import TdpClientCanvas from 'teleport/components/TdpClientCanvas'; +import { TdpClientCanvasRef } from 'teleport/components/TdpClientCanvas/TdpClientCanvas'; import cfg from 'teleport/config'; import { formatDisplayTime, StatusEnum } from 'teleport/lib/player'; -import { PlayerClient, TdpClient } from 'teleport/lib/tdp'; -import type { BitmapFrame } from 'teleport/lib/tdp/client'; +import { PlayerClient, TdpClientEvent } from 'teleport/lib/tdp'; +import { BitmapFrame } from 'teleport/lib/tdp/client'; import type { ClientScreenSpec, PngFrame } from 'teleport/lib/tdp/codec'; import { getHostName } from 'teleport/services/api'; import ProgressBar from './ProgressBar'; const reload = () => window.location.reload(); -const handleContextMenu = () => true; -const PROGRESS_BAR_ID = 'progressBarDesktop'; - -// overflow: 'hidden' is needed to prevent the canvas from outgrowing the container due to some weird css flex idiosyncracy. -// See https://gaurav5430.medium.com/css-flex-positioning-gotchas-child-expands-to-more-than-the-width-allowed-by-the-parent-799c37428dd6. -const canvasStyle = { - alignSelf: 'center', - overflow: 'hidden', -}; export const DesktopPlayer = ({ sid, @@ -56,11 +48,7 @@ export const DesktopPlayer = ({ playerStatus, statusText, time, - canvasSizeIsSet, - clientOnPngFrame, - clientOnBitmapFrame, - clientOnClientScreenSpec, clientOnWsClose, clientOnTdpError, clientOnTdpInfo, @@ -68,6 +56,85 @@ export const DesktopPlayer = ({ sid, clusterId, }); + const tdpClientCanvasRef = useRef(null); + + useEffect(() => { + if (playerClient && clientOnTdpError) { + playerClient.on(TdpClientEvent.TDP_ERROR, clientOnTdpError); + playerClient.on(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); + + return () => { + playerClient.removeListener(TdpClientEvent.TDP_ERROR, clientOnTdpError); + playerClient.removeListener( + TdpClientEvent.CLIENT_ERROR, + clientOnTdpError + ); + }; + } + }, [playerClient, clientOnTdpError]); + + useEffect(() => { + if (playerClient && clientOnTdpInfo) { + playerClient.on(TdpClientEvent.TDP_INFO, clientOnTdpInfo); + + return () => { + playerClient.removeListener(TdpClientEvent.TDP_INFO, clientOnTdpInfo); + }; + } + }, [playerClient, clientOnTdpInfo]); + + useEffect(() => { + if (playerClient && clientOnWsClose) { + playerClient.on(TdpClientEvent.WS_CLOSE, clientOnWsClose); + + return () => { + playerClient.removeListener(TdpClientEvent.WS_CLOSE, clientOnWsClose); + }; + } + }, [playerClient, clientOnWsClose]); + + useEffect(() => { + if (!playerClient) { + return; + } + const renderPngFrame = (frame: PngFrame) => + tdpClientCanvasRef.current?.renderPngFrame(frame); + playerClient.addListener(TdpClientEvent.TDP_PNG_FRAME, renderPngFrame); + + return () => { + playerClient.removeListener(TdpClientEvent.TDP_PNG_FRAME, renderPngFrame); + }; + }, [playerClient]); + + useEffect(() => { + if (!playerClient) { + return; + } + const renderBitmapFrame = (frame: BitmapFrame) => + tdpClientCanvasRef.current?.renderBitmapFrame(frame); + playerClient.addListener(TdpClientEvent.TDP_BMP_FRAME, renderBitmapFrame); + + return () => { + playerClient.removeListener( + TdpClientEvent.TDP_BMP_FRAME, + renderBitmapFrame + ); + }; + }, [playerClient]); + + useEffect(() => { + const setResolution = (spec: ClientScreenSpec) => { + tdpClientCanvasRef.current?.setResolution(spec); + }; + playerClient.on(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, setResolution); + + return () => { + playerClient.removeListener( + TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, + setResolution + ); + }; + }, [playerClient]); const isError = playerStatus === StatusEnum.ERROR || statusText !== ''; const isLoading = playerStatus === StatusEnum.LOADING; @@ -78,13 +145,6 @@ export const DesktopPlayer = ({ ? durationMs // Force progress bar to 100% when playback is complete or errored. : time; // Otherwise, use the current time. - // Hide the canvas and progress bar until the canvas' size has been fully defined. - // This prevents visual glitches at pageload where the canvas starts out small and - // then suddenly expands to its full size (moving the progress bar down with it). - const canvasAndProgressBarDisplayStyle = canvasSizeIsSet - ? {} // Canvas size is set, let TdpClientCanvas and ProgressBar use their default display styles. - : { display: 'none' }; // Canvas size is not set, hide the canvas and progress bar. - return ( {isError && } @@ -95,24 +155,9 @@ export const DesktopPlayer = ({ )} - + playerClient.setPlaySpeed(s)} toggle={() => playerClient.togglePlayPause()} - style={{ ...canvasAndProgressBarDisplayStyle }} /> ); }; -const clientOnPngFrame = ( - ctx: CanvasRenderingContext2D, - pngFrame: PngFrame -) => { - ctx.drawImage(pngFrame.data, pngFrame.left, pngFrame.top); -}; - -const clientOnBitmapFrame = ( - ctx: CanvasRenderingContext2D, - bmpFrame: BitmapFrame -) => { - ctx.putImageData(bmpFrame.image_data, bmpFrame.left, bmpFrame.top); -}; - const useDesktopPlayer = ({ clusterId, sid }) => { const [time, setTime] = useState(0); const [playerStatus, setPlayerStatus] = useState(StatusEnum.LOADING); const [statusText, setStatusText] = useState(''); - // `canvasSizeIsSet` is used to track whether the canvas' size has been fully defined. - const [canvasSizeIsSet, setCanvasSizeIsSet] = useState(false); const playerClient = useMemo(() => { const url = cfg.api.desktopPlaybackWsAddr @@ -179,35 +207,11 @@ const useDesktopPlayer = ({ clusterId, sid }) => { setStatusText(info); }, []); - const clientOnClientScreenSpec = useCallback( - (_cli: TdpClient, canvas: HTMLCanvasElement, spec: ClientScreenSpec) => { - const { width, height } = spec; - - const styledPlayer = canvas.parentElement; - const progressBar = styledPlayer.children.namedItem(PROGRESS_BAR_ID); - - const fullWidth = styledPlayer.clientWidth; - const fullHeight = styledPlayer.clientHeight - progressBar.clientHeight; - const originalAspectRatio = width / height; - const currentAspectRatio = fullWidth / fullHeight; - - if (originalAspectRatio > currentAspectRatio) { - // Use the full width of the screen and scale the height. - canvas.style.height = `${(fullWidth * height) / width}px`; - } else if (originalAspectRatio < currentAspectRatio) { - // Use the full height of the screen and scale the width. - canvas.style.width = `${(fullHeight * width) / height}px`; - } - - canvas.width = width; - canvas.height = height; - - setCanvasSizeIsSet(true); - }, - [] - ); - useEffect(() => { + if (!playerClient) { + return; + } + void playerClient.connect(); return () => { playerClient.shutdown(); }; @@ -218,11 +222,7 @@ const useDesktopPlayer = ({ clusterId, sid }) => { playerClient, playerStatus, statusText, - canvasSizeIsSet, - clientOnPngFrame, - clientOnBitmapFrame, - clientOnClientScreenSpec, clientOnWsClose, clientOnTdpError, clientOnTdpInfo, @@ -249,4 +249,5 @@ const StyledContainer = styled(Flex)` justify-content: center; width: 100%; height: 100%; + min-height: 0; `; diff --git a/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx b/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx index c1d402447e606..9c7e9a327ddbe 100644 --- a/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx +++ b/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx @@ -16,453 +16,231 @@ * along with this program. If not, see . */ -import { memo, useEffect, useRef, type CSSProperties } from 'react'; +import React, { + forwardRef, + useEffect, + useImperativeHandle, + useRef, + type CSSProperties, +} from 'react'; -import { DebouncedFunc } from 'shared/utils/highbar'; +import { Logger } from 'design/logger'; +import { debounce } from 'shared/utils/highbar'; -import { TdpClient, TdpClientEvent } from 'teleport/lib/tdp'; import { BitmapFrame } from 'teleport/lib/tdp/client'; -import type { - ClientScreenSpec, - ClipboardData, - PngFrame, -} from 'teleport/lib/tdp/codec'; +import type { PngFrame } from 'teleport/lib/tdp/codec'; -function TdpClientCanvas(props: Props) { +const logger = new Logger('TdpClientCanvas'); + +export interface TdpClientCanvasRef { + setPointer(pointer: Pointer): void; + renderPngFrame(frame: PngFrame): void; + renderBitmapFrame(frame: BitmapFrame): void; + setResolution(resolution: { width: number; height: number }): void; + clear(): void; + focus(): void; +} + +export const TdpClientCanvas = forwardRef< + TdpClientCanvasRef, + { + onKeyDown?(e: React.KeyboardEvent): void; + onKeyUp?(e: React.KeyboardEvent): void; + onBlur?(e: React.FocusEvent): void; + onMouseMove?(e: React.MouseEvent): void; + onMouseDown?(e: React.MouseEvent): void; + onMouseUp?(e: React.MouseEvent): void; + onMouseWheel?(e: WheelEvent): void; + onContextMenu?(e: React.MouseEvent): void; + /** + * Handles canvas resize events. + * + * This function is called whenever the canvas is resized, + * with a debounced delay of 250 ms to optimize performance. + */ + onResize?(e: { width: number; height: number }): void; + style?: CSSProperties; + } +>((props, ref) => { const { - client, - clientShouldConnect = false, - clientScreenSpecToRequest, - clientOnPngFrame, - clientOnBmpFrame, - clientOnClipboardData, - clientOnTdpError, - clientOnTdpWarning, - clientOnTdpInfo, - clientOnWsClose, - clientOnWsOpen, - clientOnClientScreenSpec, - canvasOnKeyDown, - canvasOnKeyUp, - canvasOnFocusOut, - canvasOnMouseMove, - canvasOnMouseDown, - canvasOnMouseUp, - canvasOnMouseWheelScroll, - canvasOnContextMenu, - windowOnResize, + onKeyDown, + onKeyUp, + onBlur, + onMouseMove, + onMouseDown, + onMouseUp, + onMouseWheel, + onContextMenu, + onResize, style, - updatePointer, } = props; const canvasRef = useRef(null); - useEffect(() => { - // Empty dependency array ensures this runs only once after initial render. - // This code will run after the component has been mounted and the canvasRef has been assigned. - const canvas = canvasRef.current; - if (canvas) { - // Make the canvas a focusable keyboard listener - // https://stackoverflow.com/a/51267699/6277051 - // https://stackoverflow.com/a/16492878/6277051 - canvas.tabIndex = -1; - canvas.style.outline = 'none'; - canvas.focus(); - } - }, []); - - useEffect(() => { - if (client && clientOnPngFrame) { - const canvas = canvasRef.current; - const ctx = canvas.getContext('2d'); - - // Buffered rendering logic - var pngBuffer: PngFrame[] = []; - const renderBuffer = () => { - if (pngBuffer.length) { - for (let i = 0; i < pngBuffer.length; i++) { - clientOnPngFrame(ctx, pngBuffer[i]); - } - pngBuffer = []; - } - requestAnimationFrame(renderBuffer); - }; - requestAnimationFrame(renderBuffer); - - const pushToPngBuffer = (pngFrame: PngFrame) => { - pngBuffer.push(pngFrame); - }; - - client.on(TdpClientEvent.TDP_PNG_FRAME, pushToPngBuffer); - - return () => { - client.removeListener(TdpClientEvent.TDP_PNG_FRAME, pushToPngBuffer); - }; - } - }, [client, clientOnPngFrame]); - - useEffect(() => { - if (client && updatePointer) { - const canvas = canvasRef.current; - const updatePointer = (pointer: { - data: ImageData | boolean; - hotspot_x?: number; - hotspot_y?: number; - }) => { - if (typeof pointer.data === 'boolean') { - canvas.style.cursor = pointer.data ? 'default' : 'none'; - return; - } - let cursor = document.createElement('canvas'); - cursor.width = pointer.data.width; - cursor.height = pointer.data.height; - cursor - .getContext('2d', { colorSpace: pointer.data.colorSpace }) - .putImageData(pointer.data, 0, 0); - if (pointer.data.width > 32 || pointer.data.height > 32) { - // scale the cursor down to at most 32px - max size fully supported by browsers - const resized = document.createElement('canvas'); - let scale = Math.min(32 / cursor.width, 32 / cursor.height); - resized.width = cursor.width * scale; - resized.height = cursor.height * scale; - - let context = resized.getContext('2d', { - colorSpace: pointer.data.colorSpace, - }); - context.scale(scale, scale); - context.drawImage(cursor, 0, 0); - cursor = resized; - } - canvas.style.cursor = `url(${cursor.toDataURL()}) ${ - pointer.hotspot_x - } ${pointer.hotspot_y}, auto`; - }; - - client.addListener(TdpClientEvent.POINTER, updatePointer); - - return () => { - client.removeListener(TdpClientEvent.POINTER, updatePointer); - }; - } - }, [client, updatePointer]); - - useEffect(() => { - if (client && clientOnBmpFrame) { - const canvas = canvasRef.current; - const ctx = canvas.getContext('2d'); - - // Buffered rendering logic - var bitmapBuffer: BitmapFrame[] = []; - const renderBuffer = () => { - if (bitmapBuffer.length) { - for (let i = 0; i < bitmapBuffer.length; i++) { - if (bitmapBuffer[i].image_data.data.length != 0) { - clientOnBmpFrame(ctx, bitmapBuffer[i]); - } - } - bitmapBuffer = []; - } - requestAnimationFrame(renderBuffer); - }; - requestAnimationFrame(renderBuffer); - - const pushToBitmapBuffer = (bmpFrame: BitmapFrame) => { - bitmapBuffer.push(bmpFrame); - }; - - client.on(TdpClientEvent.TDP_BMP_FRAME, pushToBitmapBuffer); - - return () => { - client.removeListener(TdpClientEvent.TDP_BMP_FRAME, pushToBitmapBuffer); - }; - } - }, [client, clientOnBmpFrame]); - - useEffect(() => { - if (client && clientOnClientScreenSpec) { - const canvas = canvasRef.current; - const _clientOnClientScreenSpec = (spec: ClientScreenSpec) => { - clientOnClientScreenSpec(client, canvas, spec); - }; - client.on( - TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, - _clientOnClientScreenSpec - ); - - return () => { - client.removeListener( - TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, - _clientOnClientScreenSpec - ); - }; - } - }, [client, clientOnClientScreenSpec]); - - useEffect(() => { - if (client && clientOnClipboardData) { - client.on(TdpClientEvent.TDP_CLIPBOARD_DATA, clientOnClipboardData); - - return () => { - client.removeListener( - TdpClientEvent.TDP_CLIPBOARD_DATA, - clientOnClipboardData - ); - }; - } - }, [client, clientOnClipboardData]); - - useEffect(() => { - if (client && clientOnTdpError) { - client.on(TdpClientEvent.TDP_ERROR, clientOnTdpError); - client.on(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); - - return () => { - client.removeListener(TdpClientEvent.TDP_ERROR, clientOnTdpError); - client.removeListener(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); - }; - } - }, [client, clientOnTdpError]); - - useEffect(() => { - if (client && clientOnTdpWarning) { - client.on(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); - client.on(TdpClientEvent.CLIENT_WARNING, clientOnTdpWarning); - - return () => { - client.removeListener(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); - client.removeListener( - TdpClientEvent.CLIENT_WARNING, - clientOnTdpWarning - ); - }; - } - }, [client, clientOnTdpWarning]); - - useEffect(() => { - if (client && clientOnTdpInfo) { - client.on(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - - return () => { - client.removeListener(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - }; - } - }, [client, clientOnTdpInfo]); - - useEffect(() => { - if (client && clientOnWsClose) { - client.on(TdpClientEvent.WS_CLOSE, clientOnWsClose); - - return () => { - client.removeListener(TdpClientEvent.WS_CLOSE, clientOnWsClose); - }; - } - }, [client, clientOnWsClose]); - - useEffect(() => { - if (client && clientOnWsOpen) { - client.on(TdpClientEvent.WS_OPEN, clientOnWsOpen); - - return () => { - client.removeListener(TdpClientEvent.WS_OPEN, clientOnWsOpen); - }; - } - }, [client, clientOnWsOpen]); - - useEffect(() => { - const canvas = canvasRef.current; - const _oncontextmenu = canvasOnContextMenu; - if (canvasOnContextMenu) { - canvas.oncontextmenu = _oncontextmenu; - } - - return () => { - if (canvasOnContextMenu) - canvas.removeEventListener('contextmenu', _oncontextmenu); + useImperativeHandle(ref, () => { + const renderPngFrame = makePngFrameRenderer(canvasRef.current); + const renderBimapFrame = makeBitmapFrameRenderer(canvasRef.current); + return { + setPointer: pointer => setPointer(canvasRef.current, pointer), + renderPngFrame: frame => renderPngFrame(frame), + renderBitmapFrame: frame => renderBimapFrame(frame), + setResolution: ({ width, height }) => { + const canvas = canvasRef.current; + canvas.width = width; + canvas.height = height; + logger.debug(`Canvas resolution set to ${width}x${height}.`); + }, + clear: () => { + const canvas = canvasRef.current; + canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height); + }, + focus: () => canvasRef.current.focus(), }; - }, [canvasOnContextMenu]); + }, []); useEffect(() => { - const canvas = canvasRef.current; - const _onmousemove = (e: MouseEvent) => { - canvasOnMouseMove(client, canvas, e); - }; - if (canvasOnMouseMove) { - canvas.onmousemove = _onmousemove; + if (!onResize) { + return; } - return () => { - if (canvasOnMouseMove) { - canvas.removeEventListener('mousemove', _onmousemove); + const debouncedOnResize = debounce(onResize, 250, { trailing: true }); + const observer = new ResizeObserver(([entry]) => { + if (entry) { + debouncedOnResize({ + height: entry.contentRect.height, + width: entry.contentRect.width, + }); } - }; - }, [client, canvasOnMouseMove]); - - useEffect(() => { - const canvas = canvasRef.current; - const _onmousedown = (e: MouseEvent) => { - canvasOnMouseDown(client, e); - }; - if (canvasOnMouseDown) { - canvas.onmousedown = _onmousedown; - } - - return () => { - if (canvasOnMouseDown) - canvas.removeEventListener('mousedown', _onmousedown); - }; - }, [client, canvasOnMouseDown]); - - useEffect(() => { - const canvas = canvasRef.current; - const _onmouseup = (e: MouseEvent) => { - canvasOnMouseUp(client, e); - }; - if (canvasOnMouseUp) { - canvas.onmouseup = _onmouseup; - } + }); + observer.observe(canvasRef.current); return () => { - if (canvasOnMouseUp) canvas.removeEventListener('mouseup', _onmouseup); + debouncedOnResize.cancel(); + observer.disconnect(); }; - }, [client, canvasOnMouseUp]); + }, [onResize]); + // Wheel events must be registered on a ref because React's onWheel + // uses a passive listener, so handlers are not able to call of e.preventDefault() on it. useEffect(() => { - const canvas = canvasRef.current; - const _onwheel = (e: WheelEvent) => { - canvasOnMouseWheelScroll(client, e); - }; - if (canvasOnMouseWheelScroll) { - canvas.onwheel = _onwheel; + if (!onMouseWheel) { + return; } - - return () => { - if (canvasOnMouseWheelScroll) - canvas.removeEventListener('wheel', _onwheel); - }; - }, [client, canvasOnMouseWheelScroll]); - - useEffect(() => { const canvas = canvasRef.current; - const _onkeydown = (e: KeyboardEvent) => { - canvasOnKeyDown(client, e); - }; - if (canvasOnKeyDown) { - canvas.onkeydown = _onkeydown; - } + canvas.addEventListener('wheel', onMouseWheel); + return () => canvas.removeEventListener('wheel', onMouseWheel); + }, [onMouseWheel]); - return () => { - if (canvasOnKeyDown) canvas.removeEventListener('keydown', _onkeydown); - }; - }, [client, canvasOnKeyDown]); - - useEffect(() => { - const canvas = canvasRef.current; - const _onkeyup = (e: KeyboardEvent) => { - canvasOnKeyUp(client, e); - }; - if (canvasOnKeyUp) { - canvas.onkeyup = _onkeyup; - } - - return () => { - if (canvasOnKeyUp) canvas.removeEventListener('keyup', _onkeyup); - }; - }, [client, canvasOnKeyUp]); - - useEffect(() => { - const canvas = canvasRef.current; - const _onfocusout = () => { - canvasOnFocusOut(client); - }; - if (canvasOnFocusOut) { - canvas.addEventListener('focusout', _onfocusout); - } + return ( + + ); +}); + +interface Pointer { + data: ImageData | boolean; + hotspot_x?: number; + hotspot_y?: number; +} - return () => { - if (canvasOnFocusOut) canvas.removeEventListener('focusout', _onfocusout); - }; - }, [client, canvasOnFocusOut]); +function setPointer(canvas: HTMLCanvasElement, pointer: Pointer): void { + if (typeof pointer.data === 'boolean') { + canvas.style.cursor = pointer.data ? 'default' : 'none'; + return; + } + let cursor = document.createElement('canvas'); + cursor.width = pointer.data.width; + cursor.height = pointer.data.height; + cursor + .getContext('2d', { colorSpace: pointer.data.colorSpace }) + .putImageData(pointer.data, 0, 0); + if (pointer.data.width > 32 || pointer.data.height > 32) { + // scale the cursor down to at most 32px - max size fully supported by browsers + const resized = document.createElement('canvas'); + const scale = Math.min(32 / cursor.width, 32 / cursor.height); + resized.width = cursor.width * scale; + resized.height = cursor.height * scale; + + const context = resized.getContext('2d', { + colorSpace: pointer.data.colorSpace, + }); + context.scale(scale, scale); + context.drawImage(cursor, 0, 0); + cursor = resized; + } + canvas.style.cursor = `url(${cursor.toDataURL()}) ${pointer.hotspot_x} ${pointer.hotspot_y}, auto`; +} - useEffect(() => { - if (client && windowOnResize) { - const _onresize = () => windowOnResize(client); - window.addEventListener('resize', _onresize); - return () => { - windowOnResize.cancel(); - window.removeEventListener('resize', _onresize); - }; +//TODO(gzdunek): renderBuffer is called even when the buffer is empty. +// This causes the function to run in a loop, 60 times per second +// (actually x2 because we have two frame renderers). +// Fix it in the both renderers, check if it improves performance. +function makePngFrameRenderer( + canvas: HTMLCanvasElement +): (frame: PngFrame) => void { + const ctx = canvas.getContext('2d'); + + // Buffered rendering logic + let pngBuffer: PngFrame[] = []; + + const renderBuffer = () => { + if (pngBuffer.length) { + for (let i = 0; i < pngBuffer.length; i++) { + const pngFrame = pngBuffer[i]; + ctx.drawImage(pngFrame.data, pngFrame.left, pngFrame.top); + } + pngBuffer = []; } - }, [client, windowOnResize]); - - useEffect(() => { - if (client) { - const canvas = canvasRef.current; - const _clearCanvas = () => { - const ctx = canvas.getContext('2d'); - ctx.clearRect(0, 0, canvas.width, canvas.height); - }; - client.on(TdpClientEvent.RESET, _clearCanvas); + requestAnimationFrame(renderBuffer); + }; + requestAnimationFrame(renderBuffer); - return () => { - client.removeListener(TdpClientEvent.RESET, _clearCanvas); - }; - } - }, [client]); + return frame => pngBuffer.push(frame); +} - // Call connect after all listeners have been registered - useEffect(() => { - if (client && clientShouldConnect) { - client.connect(clientScreenSpecToRequest); - return () => { - client.shutdown(); - }; +function makeBitmapFrameRenderer( + canvas: HTMLCanvasElement +): (frame: BitmapFrame) => void { + const ctx = canvas.getContext('2d'); + + // Buffered rendering logic + let bitmapBuffer: BitmapFrame[] = []; + const renderBuffer = () => { + if (bitmapBuffer.length) { + for (let i = 0; i < bitmapBuffer.length; i++) { + if (bitmapBuffer[i].image_data.data.length != 0) { + const bmpFrame = bitmapBuffer[i]; + ctx.putImageData(bmpFrame.image_data, bmpFrame.left, bmpFrame.top); + } + } + bitmapBuffer = []; } - }, [client, clientShouldConnect]); + requestAnimationFrame(renderBuffer); + }; + requestAnimationFrame(renderBuffer); - return ; + return frame => bitmapBuffer.push(frame); } - -export type Props = { - client: TdpClient; - // clientShouldConnect determines whether the TdpClientCanvas - // will try to connect to the server. - clientShouldConnect?: boolean; - // clientScreenSpecToRequest will be passed to client.connect() if - // clientShouldConnect is true. - clientScreenSpecToRequest?: ClientScreenSpec; - clientOnPngFrame?: ( - ctx: CanvasRenderingContext2D, - pngFrame: PngFrame - ) => void; - clientOnBmpFrame?: ( - ctx: CanvasRenderingContext2D, - pngFrame: BitmapFrame - ) => void; - clientOnClipboardData?: (clipboardData: ClipboardData) => void; - clientOnTdpError?: (error: Error) => void; - clientOnTdpWarning?: (warning: string) => void; - clientOnTdpInfo?: (info: string) => void; - clientOnWsClose?: (message: string) => void; - clientOnWsOpen?: () => void; - clientOnClientScreenSpec?: ( - cli: TdpClient, - canvas: HTMLCanvasElement, - spec: ClientScreenSpec - ) => void; - canvasOnKeyDown?: (cli: TdpClient, e: KeyboardEvent) => void; - canvasOnKeyUp?: (cli: TdpClient, e: KeyboardEvent) => void; - canvasOnFocusOut?: (cli: TdpClient) => void; - canvasOnMouseMove?: ( - cli: TdpClient, - canvas: HTMLCanvasElement, - e: MouseEvent - ) => void; - canvasOnMouseDown?: (cli: TdpClient, e: MouseEvent) => void; - canvasOnMouseUp?: (cli: TdpClient, e: MouseEvent) => void; - canvasOnMouseWheelScroll?: (cli: TdpClient, e: WheelEvent) => void; - canvasOnContextMenu?: () => boolean; - windowOnResize?: DebouncedFunc<(cli: TdpClient) => void>; - style?: CSSProperties; - updatePointer?: boolean; -}; - -export default memo(TdpClientCanvas); diff --git a/web/packages/teleport/src/components/TdpClientCanvas/index.ts b/web/packages/teleport/src/components/TdpClientCanvas/index.ts index 3935614f2bd87..f2fddf7fc2c40 100644 --- a/web/packages/teleport/src/components/TdpClientCanvas/index.ts +++ b/web/packages/teleport/src/components/TdpClientCanvas/index.ts @@ -16,6 +16,6 @@ * along with this program. If not, see . */ -import TdpClientCanvas from './TdpClientCanvas'; +import { TdpClientCanvas } from './TdpClientCanvas'; export default TdpClientCanvas; From a914e03c34b16be8a669b3008b086a0ac5aa91b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Zdunek Date: Thu, 6 Mar 2025 14:33:15 +0100 Subject: [PATCH 2/2] Refactor events listeners and fix a race condition when closing RDP stream (#52714) * Add `useListener` hook that makes subscribing to events easier * Expose simpler APIs for subscribing to events * Replace verbose `useEffect` listeners with `useListener` calls * Fix a race condition that caused TDP errors to be overwritten with websocket close messages * Move `clientOnTdpWarning` to `DesktopCanvas.tsx` * Move `clientOnTdpInfo` to `DesktopCanvas.tsx` * Move `clientOnTdpError` to `DesktopCanvas.tsx` * Move functions that change WS connection status to `DesktopCanvas.tsx` * Change order of functions * Do not remove listeners when shutting down the WebSocket It's the responsibility of callers who created the listeners to clean them up. Otherwise, `onWsClose` callback won't be called when we call `shutdown()`. * Fix types (cherry picked from commit 0a7d9415ab6e4685723268ed4cd843d526bfb75d) --- lib/srv/desktop/rdp/rdpclient/client.go | 3 +- .../DesktopSession/DesktopSession.story.tsx | 7 +- .../src/DesktopSession/DesktopSession.tsx | 240 +++++++----------- .../src/DesktopSession/useDesktopSession.tsx | 59 +++-- .../src/DesktopSession/useTdpClientCanvas.tsx | 63 ----- .../teleport/src/Player/DesktopPlayer.tsx | 99 ++------ web/packages/teleport/src/lib/tdp/client.ts | 95 ++++++- web/packages/teleport/src/lib/tdp/codec.ts | 6 + 8 files changed, 239 insertions(+), 333 deletions(-) diff --git a/lib/srv/desktop/rdp/rdpclient/client.go b/lib/srv/desktop/rdp/rdpclient/client.go index b086dda472ce9..4a2062d398f80 100644 --- a/lib/srv/desktop/rdp/rdpclient/client.go +++ b/lib/srv/desktop/rdp/rdpclient/client.go @@ -396,8 +396,7 @@ func (c *Client) startRustRDP(ctx context.Context) error { c.cfg.Logger.InfoContext(ctx, message) - // TODO(zmb3): convert this to severity error and ensure it renders in the UI - c.sendTDPAlert(message, tdp.SeverityInfo) + c.sendTDPAlert(message, tdp.SeverityError) return nil } diff --git a/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx b/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx index 5579875be1f2f..1d425fbfb57ab 100644 --- a/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx +++ b/web/packages/teleport/src/DesktopSession/DesktopSession.story.tsx @@ -48,8 +48,6 @@ const props: State = { }, tdpClient: fakeClient(), username: 'user', - clientOnWsOpen: () => {}, - clientOnWsClose: () => {}, wsConnection: { status: 'closed', statusText: 'websocket closed' }, setClipboardSharingState: () => {}, directorySharingState: { @@ -57,14 +55,13 @@ const props: State = { browserSupported: true, directorySelected: false, }, + addAlert: () => {}, + setWsConnection: () => {}, setDirectorySharingState: () => {}, onShareDirectory: () => {}, onCtrlAltDel: () => {}, setInitialTdpConnectionSucceeded: () => {}, clientScreenSpecToRequest: { width: 0, height: 0 }, - clientOnTdpError: () => {}, - clientOnTdpInfo: () => {}, - clientOnTdpWarning: () => {}, canvasOnKeyDown: () => {}, canvasOnKeyUp: () => {}, canvasOnMouseMove: () => {}, diff --git a/web/packages/teleport/src/DesktopSession/DesktopSession.tsx b/web/packages/teleport/src/DesktopSession/DesktopSession.tsx index c14c4f35a77e5..73505665e47e0 100644 --- a/web/packages/teleport/src/DesktopSession/DesktopSession.tsx +++ b/web/packages/teleport/src/DesktopSession/DesktopSession.tsx @@ -31,14 +31,14 @@ import { Attempt } from 'shared/hooks/useAttemptNext'; import AuthnDialog from 'teleport/components/AuthnDialog'; import TdpClientCanvas from 'teleport/components/TdpClientCanvas'; import { TdpClientCanvasRef } from 'teleport/components/TdpClientCanvas/TdpClientCanvas'; -import { TdpClientEvent } from 'teleport/lib/tdp'; -import { BitmapFrame } from 'teleport/lib/tdp/client'; -import { ClientScreenSpec, PngFrame } from 'teleport/lib/tdp/codec'; +import { useListener } from 'teleport/lib/tdp/client'; import type { MfaState } from 'teleport/lib/useMfa'; import TopBar from './TopBar'; import useDesktopSession, { clipboardSharingMessage, + defaultClipboardSharingState, + defaultDirectorySharingState, directorySharingPossible, isSharingClipboard, isSharingDirectory, @@ -67,11 +67,7 @@ export function DesktopSession(props: State) { setDirectorySharingState, setInitialTdpConnectionSucceeded, clientOnClipboardData, - clientOnTdpError, - clientOnTdpWarning, - clientOnTdpInfo, - clientOnWsClose, - clientOnWsOpen, + setWsConnection, canvasOnKeyDown, canvasOnKeyUp, canvasOnFocusOut, @@ -92,83 +88,14 @@ export function DesktopSession(props: State) { tdpConnection, wsConnection, showAnotherSessionActiveDialog, + addAlert, + setTdpConnection, } = props; const [screenState, setScreenState] = useState({ screen: 'processing', canvasState: { shouldConnect: false, shouldDisplay: false }, }); - - useEffect(() => { - if (client && clientOnClipboardData) { - client.on(TdpClientEvent.TDP_CLIPBOARD_DATA, clientOnClipboardData); - - return () => { - client.removeListener( - TdpClientEvent.TDP_CLIPBOARD_DATA, - clientOnClipboardData - ); - }; - } - }, [client, clientOnClipboardData]); - - useEffect(() => { - if (client && clientOnTdpError) { - client.on(TdpClientEvent.TDP_ERROR, clientOnTdpError); - client.on(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); - - return () => { - client.removeListener(TdpClientEvent.TDP_ERROR, clientOnTdpError); - client.removeListener(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); - }; - } - }, [client, clientOnTdpError]); - - useEffect(() => { - if (client && clientOnTdpWarning) { - client.on(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); - client.on(TdpClientEvent.CLIENT_WARNING, clientOnTdpWarning); - - return () => { - client.removeListener(TdpClientEvent.TDP_WARNING, clientOnTdpWarning); - client.removeListener( - TdpClientEvent.CLIENT_WARNING, - clientOnTdpWarning - ); - }; - } - }, [client, clientOnTdpWarning]); - - useEffect(() => { - if (client && clientOnTdpInfo) { - client.on(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - - return () => { - client.removeListener(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - }; - } - }, [client, clientOnTdpInfo]); - - useEffect(() => { - if (client && clientOnWsClose) { - client.on(TdpClientEvent.WS_CLOSE, clientOnWsClose); - - return () => { - client.removeListener(TdpClientEvent.WS_CLOSE, clientOnWsClose); - }; - } - }, [client, clientOnWsClose]); - - useEffect(() => { - if (client && clientOnWsOpen) { - client.on(TdpClientEvent.WS_OPEN, clientOnWsOpen); - - return () => { - client.removeListener(TdpClientEvent.WS_OPEN, clientOnWsOpen); - }; - } - }, [client, clientOnWsOpen]); - const { shouldConnect } = screenState.canvasState; // Call connect after all listeners have been registered useEffect(() => { @@ -201,19 +128,6 @@ export function DesktopSession(props: State) { ]); const tdpClientCanvasRef = useRef(null); - - useEffect(() => { - if (!client) { - return; - } - const setPointer = tdpClientCanvasRef.current?.setPointer; - client.addListener(TdpClientEvent.POINTER, setPointer); - - return () => { - client.removeListener(TdpClientEvent.POINTER, setPointer); - }; - }, [client]); - const onInitialTdpConnectionSucceeded = useCallback(() => { setInitialTdpConnectionSucceeded(() => { // TODO(gzdunek): This callback is a temporary fix for focusing the canvas. @@ -226,63 +140,86 @@ export function DesktopSession(props: State) { }); }, [setInitialTdpConnectionSucceeded]); - useEffect(() => { - if (!client) { - return; - } - const renderFrame = (frame: PngFrame) => { - onInitialTdpConnectionSucceeded(); - tdpClientCanvasRef.current?.renderPngFrame(frame); - }; - client.addListener(TdpClientEvent.TDP_PNG_FRAME, renderFrame); - - return () => { - client.removeListener(TdpClientEvent.TDP_PNG_FRAME, renderFrame); - }; - }, [client, onInitialTdpConnectionSucceeded]); - - useEffect(() => { - if (!client) { - return; - } - const renderFrame = (frame: BitmapFrame) => { - onInitialTdpConnectionSucceeded(); - tdpClientCanvasRef.current?.renderBitmapFrame(frame); - }; - client.addListener(TdpClientEvent.TDP_BMP_FRAME, renderFrame); - - return () => { - client.removeListener(TdpClientEvent.TDP_BMP_FRAME, renderFrame); - }; - }, [client, onInitialTdpConnectionSucceeded]); - - useEffect(() => { - if (!client) { - return; - } - const clear = () => tdpClientCanvasRef.current?.clear(); - client.addListener(TdpClientEvent.RESET, clear); - - return () => { - client.removeListener(TdpClientEvent.RESET, clear); - }; - }, [client]); + useListener(client?.onClipboardData, clientOnClipboardData); + + const handleFatalError = useCallback( + (error: Error) => { + setDirectorySharingState(defaultDirectorySharingState); + setClipboardSharingState(defaultClipboardSharingState); + setTdpConnection({ + status: 'failed', + statusText: error.message || error.toString(), + }); + }, + [setClipboardSharingState, setDirectorySharingState, setTdpConnection] + ); + useListener(client?.onError, handleFatalError); + useListener(client?.onClientError, handleFatalError); + + const addWarning = useCallback( + (warning: string) => { + addAlert({ + content: warning, + severity: 'warn', + }); + }, + [addAlert] + ); + useListener(client?.onWarning, addWarning); + useListener(client?.onClientWarning, addWarning); + + useListener( + client?.onInfo, + useCallback( + info => { + addAlert({ + content: info, + severity: 'info', + }); + }, + [addAlert] + ) + ); - useEffect(() => { - if (!client) { - return; - } - const setResolution = (spec: ClientScreenSpec) => - tdpClientCanvasRef.current?.setResolution(spec); - client.addListener(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, setResolution); + useListener( + client?.onWsClose, + useCallback( + statusText => { + setWsConnection({ status: 'closed', statusText }); + }, + [setWsConnection] + ) + ); + useListener( + client?.onWsOpen, + useCallback(() => { + setWsConnection({ status: 'open' }); + }, [setWsConnection]) + ); - return () => { - client.removeListener( - TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, - setResolution - ); - }; - }, [client]); + useListener(client?.onPointer, tdpClientCanvasRef.current?.setPointer); + useListener( + client?.onPngFrame, + useCallback( + frame => { + onInitialTdpConnectionSucceeded(); + tdpClientCanvasRef.current?.renderPngFrame(frame); + }, + [onInitialTdpConnectionSucceeded] + ) + ); + useListener( + client?.onBmpFrame, + useCallback( + frame => { + onInitialTdpConnectionSucceeded(); + tdpClientCanvasRef.current?.renderBitmapFrame(frame); + }, + [onInitialTdpConnectionSucceeded] + ) + ); + useListener(client?.onReset, tdpClientCanvasRef.current?.clear); + useListener(client?.onScreenSpec, tdpClientCanvasRef.current?.setResolution); return ( . */ -import { Dispatch, SetStateAction, useEffect, useMemo, useState } from 'react'; +import { + Dispatch, + SetStateAction, + useCallback, + useEffect, + useMemo, + useState, +} from 'react'; import { useParams } from 'react-router'; import type { NotificationItem } from 'shared/components/Notification'; @@ -115,6 +122,12 @@ export default function useDesktopSession() { const onRemoveAlert = (id: string) => { setAlerts(prevState => prevState.filter(alert => alert.id !== id)); }; + const addAlert = useCallback((alert: Omit) => { + setAlerts(prevState => [ + ...prevState, + { ...alert, id: crypto.randomUUID() }, + ]); + }, []); const clientCanvasProps = useTdpClientCanvas({ username, @@ -149,39 +162,33 @@ export default function useDesktopSession() { ...prevState, directorySelected: false, })); - setAlerts(prevState => [ - ...prevState, - { - id: crypto.randomUUID(), - severity: 'warn', - content: 'Failed to open the directory picker: ' + e.message, - }, - ]); + addAlert({ + severity: 'warn', + content: 'Failed to open the directory picker: ' + e.message, + }); }); } catch (e) { setDirectorySharingState(prevState => ({ ...prevState, directorySelected: false, })); - setAlerts(prevState => [ - ...prevState, - { - id: crypto.randomUUID(), - severity: 'warn', - // This is a gross error message, but should be infrequent enough that its worth just telling - // the user the likely problem, while also displaying the error message just in case that's not it. - // In a perfect world, we could check for which error message this is and display - // context appropriate directions. - content: - 'Encountered an error while attempting to share a directory: ' + + addAlert({ + severity: 'warn', + // This is a gross error message, but should be infrequent enough that its worth just telling + // the user the likely problem, while also displaying the error message just in case that's not it. + // In a perfect world, we could check for which error message this is and display + // context appropriate directions. + content: { + title: 'Encountered an error while attempting to share a directory: ', + description: e.message + '. \n\nYour user role supports directory sharing over desktop access, \ - however this feature is only available by default on some Chromium \ - based browsers like Google Chrome or Microsoft Edge. Brave users can \ - use the feature by navigating to brave://flags/#file-system-access-api \ - and selecting "Enable". If you\'re not already, please switch to a supported browser.', + however this feature is only available by default on some Chromium \ + based browsers like Google Chrome or Microsoft Edge. Brave users can \ + use the feature by navigating to brave://flags/#file-system-access-api \ + and selecting "Enable". If you\'re not already, please switch to a supported browser.', }, - ]); + }); } }; @@ -212,6 +219,8 @@ export default function useDesktopSession() { onCtrlAltDel, alerts, onRemoveAlert, + addAlert, + setWsConnection, ...clientCanvasProps, }; } diff --git a/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx b/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx index 3a7c5fe987270..332490e61cbfb 100644 --- a/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx +++ b/web/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx @@ -33,8 +33,6 @@ import { TopBarHeight } from './TopBar'; import { clipboardSharingPossible, ClipboardSharingState, - defaultClipboardSharingState, - defaultDirectorySharingState, DirectorySharingState, isSharingClipboard, Setter, @@ -52,11 +50,7 @@ export default function useTdpClientCanvas(props: Props) { desktopName, clusterId, setTdpConnection, - setWsConnection, clipboardSharingState, - setClipboardSharingState, - setDirectorySharingState, - setAlerts, } = props; const [tdpClient, setTdpClient] = useState(null); const initialTdpConnectionSucceeded = useRef(false); @@ -107,58 +101,6 @@ export default function useTdpClientCanvas(props: Props) { } }; - // Default TdpClientEvent.TDP_ERROR and TdpClientEvent.CLIENT_ERROR handler - const clientOnTdpError = (error: Error) => { - setDirectorySharingState(defaultDirectorySharingState); - setClipboardSharingState(defaultClipboardSharingState); - setTdpConnection(prevState => { - // Sometimes when a connection closes due to an error, we get a cascade of - // errors. Here we update the status only if it's not already 'failed', so - // that the first error message (which is usually the most informative) is - // displayed to the user. - if (prevState.status !== 'failed') { - return { - status: 'failed', - statusText: error.message || error.toString(), - }; - } - return prevState; - }); - }; - - // Default TdpClientEvent.TDP_WARNING and TdpClientEvent.CLIENT_WARNING handler - const clientOnTdpWarning = (warning: string) => { - setAlerts(prevState => { - return [ - ...prevState, - { - content: warning, - severity: 'warn', - id: crypto.randomUUID(), - }, - ]; - }); - }; - - // TODO(zmb3): this is not what an info-level alert should do. - // rename it to something like onGracefulDisconnect - const clientOnTdpInfo = (info: string) => { - setDirectorySharingState(defaultDirectorySharingState); - setClipboardSharingState(defaultClipboardSharingState); - setTdpConnection({ - status: '', // gracefully disconnecting - statusText: info, - }); - }; - - const clientOnWsClose = (statusText: string) => { - setWsConnection({ status: 'closed', statusText }); - }; - - const clientOnWsOpen = () => { - setWsConnection({ status: 'open' }); - }; - const canvasOnKeyDown = (e: React.KeyboardEvent) => { keyboardHandler.current.handleKeyboardEvent({ cli: tdpClient, @@ -257,12 +199,7 @@ export default function useTdpClientCanvas(props: Props) { tdpClient, clientScreenSpecToRequest: getDisplaySize(), setInitialTdpConnectionSucceeded, - clientOnTdpError, clientOnClipboardData, - clientOnWsClose, - clientOnWsOpen, - clientOnTdpWarning, - clientOnTdpInfo, canvasOnKeyDown, canvasOnKeyUp, canvasOnFocusOut, diff --git a/web/packages/teleport/src/Player/DesktopPlayer.tsx b/web/packages/teleport/src/Player/DesktopPlayer.tsx index dcfbeec0b98d4..db8cc0b273c23 100644 --- a/web/packages/teleport/src/Player/DesktopPlayer.tsx +++ b/web/packages/teleport/src/Player/DesktopPlayer.tsx @@ -25,9 +25,8 @@ import TdpClientCanvas from 'teleport/components/TdpClientCanvas'; import { TdpClientCanvasRef } from 'teleport/components/TdpClientCanvas/TdpClientCanvas'; import cfg from 'teleport/config'; import { formatDisplayTime, StatusEnum } from 'teleport/lib/player'; -import { PlayerClient, TdpClientEvent } from 'teleport/lib/tdp'; -import { BitmapFrame } from 'teleport/lib/tdp/client'; -import type { ClientScreenSpec, PngFrame } from 'teleport/lib/tdp/codec'; +import { PlayerClient } from 'teleport/lib/tdp'; +import { useListener } from 'teleport/lib/tdp/client'; import { getHostName } from 'teleport/services/api'; import ProgressBar from './ProgressBar'; @@ -58,83 +57,23 @@ export const DesktopPlayer = ({ }); const tdpClientCanvasRef = useRef(null); - useEffect(() => { - if (playerClient && clientOnTdpError) { - playerClient.on(TdpClientEvent.TDP_ERROR, clientOnTdpError); - playerClient.on(TdpClientEvent.CLIENT_ERROR, clientOnTdpError); - - return () => { - playerClient.removeListener(TdpClientEvent.TDP_ERROR, clientOnTdpError); - playerClient.removeListener( - TdpClientEvent.CLIENT_ERROR, - clientOnTdpError - ); - }; - } - }, [playerClient, clientOnTdpError]); - - useEffect(() => { - if (playerClient && clientOnTdpInfo) { - playerClient.on(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - - return () => { - playerClient.removeListener(TdpClientEvent.TDP_INFO, clientOnTdpInfo); - }; - } - }, [playerClient, clientOnTdpInfo]); - - useEffect(() => { - if (playerClient && clientOnWsClose) { - playerClient.on(TdpClientEvent.WS_CLOSE, clientOnWsClose); - - return () => { - playerClient.removeListener(TdpClientEvent.WS_CLOSE, clientOnWsClose); - }; - } - }, [playerClient, clientOnWsClose]); - - useEffect(() => { - if (!playerClient) { - return; - } - const renderPngFrame = (frame: PngFrame) => - tdpClientCanvasRef.current?.renderPngFrame(frame); - playerClient.addListener(TdpClientEvent.TDP_PNG_FRAME, renderPngFrame); - - return () => { - playerClient.removeListener(TdpClientEvent.TDP_PNG_FRAME, renderPngFrame); - }; - }, [playerClient]); - - useEffect(() => { - if (!playerClient) { - return; - } - const renderBitmapFrame = (frame: BitmapFrame) => - tdpClientCanvasRef.current?.renderBitmapFrame(frame); - playerClient.addListener(TdpClientEvent.TDP_BMP_FRAME, renderBitmapFrame); - - return () => { - playerClient.removeListener( - TdpClientEvent.TDP_BMP_FRAME, - renderBitmapFrame - ); - }; - }, [playerClient]); - - useEffect(() => { - const setResolution = (spec: ClientScreenSpec) => { - tdpClientCanvasRef.current?.setResolution(spec); - }; - playerClient.on(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, setResolution); - - return () => { - playerClient.removeListener( - TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, - setResolution - ); - }; - }, [playerClient]); + useListener(playerClient?.onError, clientOnTdpError); + useListener(playerClient?.onClientError, clientOnTdpError); + useListener(playerClient?.onClientError, clientOnTdpError); + useListener(playerClient?.onInfo, clientOnTdpInfo); + useListener(playerClient?.onWsClose, clientOnWsClose); + useListener( + playerClient?.onPngFrame, + tdpClientCanvasRef.current?.renderPngFrame + ); + useListener( + playerClient?.onBmpFrame, + tdpClientCanvasRef.current?.renderBitmapFrame + ); + useListener( + playerClient?.onScreenSpec, + tdpClientCanvasRef.current?.setResolution + ); const isError = playerStatus === StatusEnum.ERROR || statusText !== ''; const isLoading = playerStatus === StatusEnum.LOADING; diff --git a/web/packages/teleport/src/lib/tdp/client.ts b/web/packages/teleport/src/lib/tdp/client.ts index 83250b8bddbc6..0014ad75723cb 100644 --- a/web/packages/teleport/src/lib/tdp/client.ts +++ b/web/packages/teleport/src/lib/tdp/client.ts @@ -16,6 +16,8 @@ * along with this program. If not, see . */ +import { useEffect } from 'react'; + import Logger from 'shared/libs/logger'; import init, { @@ -30,6 +32,7 @@ import { MfaChallengeResponse } from 'teleport/services/mfa'; import Codec, { FileType, MessageType, + PointerData, Severity, SharedDirectoryErrCode, type ButtonState, @@ -152,6 +155,71 @@ export default class Client extends EventEmitterMfaSender { }; } + onClientError = (listener: (error: Error) => void) => { + this.on(TdpClientEvent.CLIENT_ERROR, listener); + return () => this.off(TdpClientEvent.CLIENT_ERROR, listener); + }; + + onClientWarning = (listener: (warningMessage: string) => void) => { + this.on(TdpClientEvent.CLIENT_WARNING, listener); + return () => this.off(TdpClientEvent.CLIENT_WARNING, listener); + }; + + onError = (listener: (error: Error) => void) => { + this.on(TdpClientEvent.TDP_ERROR, listener); + return () => this.off(TdpClientEvent.TDP_ERROR, listener); + }; + + onInfo = (listener: (info: string) => void) => { + this.on(TdpClientEvent.TDP_INFO, listener); + return () => this.off(TdpClientEvent.TDP_INFO, listener); + }; + + onReset = (listener: () => void) => { + this.on(TdpClientEvent.RESET, listener); + return () => this.off(TdpClientEvent.RESET, listener); + }; + + onBmpFrame = (listener: (bmpFrame: BitmapFrame) => void) => { + this.on(TdpClientEvent.TDP_BMP_FRAME, listener); + return () => this.off(TdpClientEvent.TDP_BMP_FRAME, listener); + }; + + onPngFrame = (listener: (pngFrame: PngFrame) => void) => { + this.on(TdpClientEvent.TDP_PNG_FRAME, listener); + return () => this.off(TdpClientEvent.TDP_PNG_FRAME, listener); + }; + + onPointer = (listener: (pointerData: PointerData) => void) => { + this.on(TdpClientEvent.POINTER, listener); + return () => this.off(TdpClientEvent.POINTER, listener); + }; + + onWarning = (listener: (warningMessage: string) => void) => { + this.on(TdpClientEvent.TDP_WARNING, listener); + return () => this.off(TdpClientEvent.TDP_WARNING, listener); + }; + + onWsClose = (listener: (message: string) => void) => { + this.on(TdpClientEvent.WS_CLOSE, listener); + return () => this.off(TdpClientEvent.WS_CLOSE, listener); + }; + + onWsOpen = (listener: () => void) => { + this.on(TdpClientEvent.WS_OPEN, listener); + return () => this.off(TdpClientEvent.WS_OPEN, listener); + }; + + onClipboardData = (listener: (clipboardData: ClipboardData) => void) => { + this.on(TdpClientEvent.TDP_CLIPBOARD_DATA, listener); + return () => this.off(TdpClientEvent.TDP_CLIPBOARD_DATA, listener); + }; + + onScreenSpec = (listener: (spec: ClientScreenSpec) => void) => { + this.on(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, listener); + return () => this.off(TdpClientEvent.TDP_CLIENT_SCREEN_SPEC, listener); + }; + private async initWasm() { // select the wasm log level let wasmLogLevel = LogType.OFF; @@ -704,7 +772,13 @@ export default class Client extends EventEmitterMfaSender { ) { this.logger.error(err); this.emit(errType, err); - this.socket?.close(); + // All errors are fatal, meaning that we are closing the connection after they happen. + // To prevent overwriting such error with our close handler, remove it before + // closing the connection. + if (this.socket) { + this.socket.onclose = null; + this.socket.close(); + } } // Emits a warning event, but keeps the socket open. @@ -721,13 +795,9 @@ export default class Client extends EventEmitterMfaSender { this.emit(TdpClientEvent.TDP_INFO, info); } - // Ensures full cleanup of this object. - // Note that it removes all listeners first and then cleans up the socket, - // so don't call this if your calling object is relying on listeners. // It's safe to call this multiple times, calls subsequent to the first call // will simply do nothing. shutdown(closeCode = WebsocketCloseCode.NORMAL) { - this.removeAllListeners(); this.socket?.close(closeCode); } } @@ -738,3 +808,18 @@ export type BitmapFrame = { left: number; image_data: ImageData; }; + +export function useListener( + emitter: (callback: (...args: T) => void) => () => void | undefined, + listener: ((...args: T) => void) | undefined +) { + useEffect(() => { + if (!emitter) { + return; + } + const unregister = emitter((...args) => listener?.(...args)); + return () => { + unregister(); + }; + }, [emitter, listener]); +} diff --git a/web/packages/teleport/src/lib/tdp/codec.ts b/web/packages/teleport/src/lib/tdp/codec.ts index 064a3bae97abb..c57ff4f89d7dd 100644 --- a/web/packages/teleport/src/lib/tdp/codec.ts +++ b/web/packages/teleport/src/lib/tdp/codec.ts @@ -78,6 +78,12 @@ export type ClientScreenSpec = { height: number; }; +export type PointerData = { + data: ImageData | boolean; + hotspot_x?: number; + hotspot_y?: number; +}; + // | message type (2) | left uint32 | top uint32 | right uint32 | bottom uint32 | data []byte | // https://github.com/gravitational/teleport/blob/master/rfd/0037-desktop-access-protocol.md#2---png-frame export type PngFrame = {