[Flyout System] Fix circular dependency of effect and performance issues

packages/eui/src/components/flyout/manager/layout_mode.test.tsx

@@ -616,6 +616,150 @@ describe('layout_mode', () => {
       // The hook should re-evaluate with new window width
       expect(mockWindow.requestAnimationFrame).toHaveBeenCalled();
     });
+
+    describe('resize listener optimization', () => {
+      it('should NOT attach resize listener when there is no child flyout', () => {
+        // Set up session with only parent flyout (no child)
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: 'main-1',
+          childFlyoutId: null,
+        });
+
+        mockUseCurrentChildFlyout.mockReturnValue(null);
+
+        const mockDispatch = jest.fn();
+        mockUseFlyoutManager.mockReturnValue({
+          state: {
+            layoutMode: LAYOUT_MODE_SIDE_BY_SIDE,
+          },
+          dispatch: mockDispatch,
+        });
+
+        render(<TestComponent />);
+
+        // Resize listener should NOT be attached when there's no child flyout
+        expect(mockWindow.addEventListener).not.toHaveBeenCalledWith(
+          'resize',
+          expect.any(Function)
+        );
+      });
+
+      it('should attach resize listener when there is a child flyout', () => {
+        // Set up session with both parent and child flyouts
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: 'main-1',
+          childFlyoutId: 'child-1',
+        });
+
+        mockUseCurrentChildFlyout.mockReturnValue({
+          flyoutId: 'child-1',
+          level: 'child',
+          size: 's',
+        });
+
+        const mockDispatch = jest.fn();
+        mockUseFlyoutManager.mockReturnValue({
+          state: {
+            layoutMode: LAYOUT_MODE_SIDE_BY_SIDE,
+          },
+          dispatch: mockDispatch,
+        });
+
+        render(<TestComponent />);
+
+        // Resize listener SHOULD be attached when there's a child flyout
+        expect(mockWindow.addEventListener).toHaveBeenCalledWith(
+          'resize',
+          expect.any(Function)
+        );
+      });
+
+      it('should remove and re-add resize listener when child flyout changes', () => {
+        // Start with no child flyout
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: 'main-1',
+          childFlyoutId: null,
+        });
+
+        mockUseCurrentChildFlyout.mockReturnValue(null);
+
+        const mockDispatch = jest.fn();
+        mockUseFlyoutManager.mockReturnValue({
+          state: {
+            layoutMode: LAYOUT_MODE_SIDE_BY_SIDE,
+          },
+          dispatch: mockDispatch,
+        });
+
+        const { rerender } = render(<TestComponent />);
+
+        // Should not have listener initially
+        expect(mockWindow.addEventListener).not.toHaveBeenCalled();
+
+        // Now add a child flyout
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: 'main-1',
+          childFlyoutId: 'child-1',
+        });
+
+        mockUseCurrentChildFlyout.mockReturnValue({
+          flyoutId: 'child-1',
+          level: 'child',
+          size: 's',
+        });
+
+        rerender(<TestComponent />);
+
+        // Should now have attached the listener
+        expect(mockWindow.addEventListener).toHaveBeenCalledWith(
+          'resize',
+          expect.any(Function)
+        );
+
+        // Now remove the child flyout
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: 'main-1',
+          childFlyoutId: null,
+        });
+
+        mockUseCurrentChildFlyout.mockReturnValue(null);
+
+        rerender(<TestComponent />);
+
+        // Should have removed the listener
+        expect(mockWindow.removeEventListener).toHaveBeenCalledWith(
+          'resize',
+          expect.any(Function)
+        );
+      });
+
+      it('should NOT attach resize listener when there is no parent flyout', () => {
+        // Set up session with no flyouts at all
+        mockUseCurrentSession.mockReturnValue({
+          mainFlyoutId: null,
+          childFlyoutId: null,
+        });
+
+        mockUseCurrentMainFlyout.mockReturnValue(null);
+        mockUseCurrentChildFlyout.mockReturnValue(null);
+
+        const mockDispatch = jest.fn();
+        mockUseFlyoutManager.mockReturnValue({
+          state: {
+            layoutMode: LAYOUT_MODE_SIDE_BY_SIDE,
+          },
+          dispatch: mockDispatch,
+        });
+
+        render(<TestComponent />);
+
+        // Resize listener should NOT be attached when there are no flyouts
+        expect(mockWindow.addEventListener).not.toHaveBeenCalledWith(
+          'resize',
+          expect.any(Function)
+        );
+      });
+    });
   });
 
   describe('useApplyFlyoutLayoutMode with fill size', () => {

packages/eui/src/components/flyout/manager/layout_mode.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import React, { useEffect, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { useEuiTheme } from '../../../services';
 import { setLayoutMode } from './actions';
 import {
@@ -42,17 +42,28 @@ export const useApplyFlyoutLayoutMode = () => {
     typeof window !== 'undefined' ? window.innerWidth : Infinity
   );
 
-  const setMode = React.useCallback(
+  // Only set up resize listener when there's an active flyout
+  const hasFlyouts = Boolean(parentFlyoutId);
+  // Layout mode only matters when there's a child flyout to position
+  const hasChildFlyout = Boolean(childFlyoutId);
+
+  // Extract specific context values to avoid depending on the entire context object
+  // which gets a new reference on every state update
+  const dispatch = context?.dispatch;
+  const currentLayoutMode = context?.state?.layoutMode;
+
+  const setMode = useCallback(
     (layoutMode: EuiFlyoutLayoutMode) => {
-      if (context?.dispatch && layoutMode !== context.state.layoutMode) {
-        context.dispatch(setLayoutMode(layoutMode));
+      if (dispatch) {
+        dispatch(setLayoutMode(layoutMode));
       }
     },
-    [context]
+    [dispatch]
   );
 
   useEffect(() => {
-    if (typeof window === 'undefined') {
+    // Skip if no child flyout - layout mode only matters with multiple flyouts
+    if (typeof window === 'undefined' || !hasChildFlyout) {
       return;
     }
 
@@ -73,15 +84,15 @@ export const useApplyFlyoutLayoutMode = () => {
       }
       window.removeEventListener('resize', handleResize);
     };
-  }, []);
+  }, [hasChildFlyout]);
 
-  useEffect(() => {
-    if (!context) {
-      return;
+  // Calculate the desired layout mode based on current conditions
+  const desiredLayoutMode = useMemo(() => {
+    // Skip calculation if no flyouts open
+    if (!hasFlyouts) {
+      return null;
     }
 
-    const currentLayoutMode = context.state.layoutMode;
-
     // Thresholds to prevent thrashing near the breakpoint.
     const THRESHOLD_TO_SIDE_BY_SIDE = 85;
     const THRESHOLD_TO_STACKED = 95;
@@ -92,16 +103,11 @@ export const useApplyFlyoutLayoutMode = () => {
     // `composeFlyoutSizing` in `flyout.styles.ts` multiplied
     // by 2 (open flyouts side-by-side).
     if (windowWidth < Math.round(euiTheme.breakpoint.s * 1.4)) {
-      if (currentLayoutMode !== LAYOUT_MODE_STACKED) {
-        setMode(LAYOUT_MODE_STACKED);
-      }
-      return;
+      return LAYOUT_MODE_STACKED;
     }
 
     if (!childFlyoutId) {
-      if (currentLayoutMode !== LAYOUT_MODE_SIDE_BY_SIDE)
-        setMode(LAYOUT_MODE_SIDE_BY_SIDE);
-      return;
+      return LAYOUT_MODE_SIDE_BY_SIDE;
     }
 
     let parentWidthValue = parentWidth;
@@ -116,54 +122,49 @@ export const useApplyFlyoutLayoutMode = () => {
     }
 
     if (!parentWidthValue || !childWidthValue) {
-      if (currentLayoutMode !== LAYOUT_MODE_SIDE_BY_SIDE)
-        setMode(LAYOUT_MODE_SIDE_BY_SIDE);
-      return;
+      return LAYOUT_MODE_SIDE_BY_SIDE;
     }
 
     const combinedWidth = parentWidthValue + childWidthValue;
     const combinedWidthPercentage = (combinedWidth / windowWidth) * 100;
-    let newLayoutMode: EuiFlyoutLayoutMode;
 
     // Handle fill size flyouts: keep layout as side-by-side when fill flyout is present
     // This allows fill flyouts to dynamically calculate their width based on the other in the pair
     if (parentFlyout?.size === 'fill' || childFlyout?.size === 'fill') {
       // For fill flyouts, we want to maintain side-by-side layout to enable dynamic width calculation
       // Only stack if the viewport is too small (below the small breakpoint)
       if (windowWidth >= Math.round(euiTheme.breakpoint.s * 1.4)) {
-        if (currentLayoutMode !== LAYOUT_MODE_SIDE_BY_SIDE) {
-          setMode(LAYOUT_MODE_SIDE_BY_SIDE);
-        }
-        return;
+        return LAYOUT_MODE_SIDE_BY_SIDE;
       }
     }
 
     if (currentLayoutMode === LAYOUT_MODE_STACKED) {
-      newLayoutMode =
-        combinedWidthPercentage <= THRESHOLD_TO_SIDE_BY_SIDE
-          ? LAYOUT_MODE_SIDE_BY_SIDE
-          : LAYOUT_MODE_STACKED;
+      return combinedWidthPercentage <= THRESHOLD_TO_SIDE_BY_SIDE
+        ? LAYOUT_MODE_SIDE_BY_SIDE
+        : LAYOUT_MODE_STACKED;
     } else {
-      newLayoutMode =
-        combinedWidthPercentage >= THRESHOLD_TO_STACKED
-          ? LAYOUT_MODE_STACKED
-          : LAYOUT_MODE_SIDE_BY_SIDE;
-    }
-
-    if (currentLayoutMode !== newLayoutMode) {
-      setMode(newLayoutMode);
+      return combinedWidthPercentage >= THRESHOLD_TO_STACKED
+        ? LAYOUT_MODE_STACKED
+        : LAYOUT_MODE_SIDE_BY_SIDE;
     }
   }, [
+    hasFlyouts,
     windowWidth,
-    context,
+    euiTheme,
+    childFlyoutId,
     parentWidth,
-    setMode,
     childWidth,
-    childFlyoutId,
     parentFlyout?.size,
     childFlyout?.size,
-    euiTheme,
+    currentLayoutMode,
   ]);
+
+  // Apply the desired layout mode when it differs from the current mode
+  useEffect(() => {
+    if (desiredLayoutMode && currentLayoutMode !== desiredLayoutMode) {
+      setMode(desiredLayoutMode);
+    }
+  }, [desiredLayoutMode, currentLayoutMode, setMode]);
 };
 
 /** Convert a flyout `size` value to a pixel width using theme breakpoints. */