diff --git a/packages/block-library/src/navigation/edit/index.js b/packages/block-library/src/navigation/edit/index.js
index 7d79e0f49880c5..e232be2e0fe5d1 100644
--- a/packages/block-library/src/navigation/edit/index.js
+++ b/packages/block-library/src/navigation/edit/index.js
@@ -73,6 +73,116 @@ import AccessibleDescription from './accessible-description';
 import AccessibleMenuDescription from './accessible-menu-description';
 import { unlock } from '../../lock-unlock';
 
+function ColorTools( {
+	textColor,
+	setTextColor,
+	backgroundColor,
+	setBackgroundColor,
+	overlayTextColor,
+	setOverlayTextColor,
+	overlayBackgroundColor,
+	setOverlayBackgroundColor,
+	clientId,
+	navRef,
+} ) {
+	const [ detectedBackgroundColor, setDetectedBackgroundColor ] = useState();
+	const [ detectedColor, setDetectedColor ] = useState();
+	const [
+		detectedOverlayBackgroundColor,
+		setDetectedOverlayBackgroundColor,
+	] = useState();
+	const [ detectedOverlayColor, setDetectedOverlayColor ] = useState();
+	// Turn on contrast checker for web only since it's not supported on mobile yet.
+	const enableContrastChecking = Platform.OS === 'web';
+	useEffect( () => {
+		if ( ! enableContrastChecking ) {
+			return;
+		}
+		detectColors(
+			navRef.current,
+			setDetectedColor,
+			setDetectedBackgroundColor
+		);
+
+		const subMenuElement = navRef.current?.querySelector(
+			'[data-type="core/navigation-submenu"] [data-type="core/navigation-link"]'
+		);
+
+		if ( ! subMenuElement ) {
+			return;
+		}
+
+		// Only detect submenu overlay colors if they have previously been explicitly set.
+		// This avoids the contrast checker from reporting on inherited submenu colors and
+		// showing the contrast warning twice.
+		if ( overlayTextColor.color || overlayBackgroundColor.color ) {
+			detectColors(
+				subMenuElement,
+				setDetectedOverlayColor,
+				setDetectedOverlayBackgroundColor
+			);
+		}
+	}, [
+		enableContrastChecking,
+		overlayTextColor.color,
+		overlayBackgroundColor.color,
+		navRef,
+	] );
+	const colorGradientSettings = useMultipleOriginColorsAndGradients();
+	if ( ! colorGradientSettings.hasColorsOrGradients ) {
+		return null;
+	}
+	return (
+		<>
+			<ColorGradientSettingsDropdown
+				__experimentalIsRenderedInSidebar
+				settings={ [
+					{
+						colorValue: textColor.color,
+						label: __( 'Text' ),
+						onColorChange: setTextColor,
+						resetAllFilter: () => setTextColor(),
+					},
+					{
+						colorValue: backgroundColor.color,
+						label: __( 'Background' ),
+						onColorChange: setBackgroundColor,
+						resetAllFilter: () => setBackgroundColor(),
+					},
+					{
+						colorValue: overlayTextColor.color,
+						label: __( 'Submenu & overlay text' ),
+						onColorChange: setOverlayTextColor,
+						resetAllFilter: () => setOverlayTextColor(),
+					},
+					{
+						colorValue: overlayBackgroundColor.color,
+						label: __( 'Submenu & overlay background' ),
+						onColorChange: setOverlayBackgroundColor,
+						resetAllFilter: () => setOverlayBackgroundColor(),
+					},
+				] }
+				panelId={ clientId }
+				{ ...colorGradientSettings }
+				gradients={ [] }
+				disableCustomGradients
+			/>
+			{ enableContrastChecking && (
+				<>
+					<ContrastChecker
+						backgroundColor={ detectedBackgroundColor }
+						textColor={ detectedColor }
+					/>
+					<ContrastChecker
+						backgroundColor={ detectedOverlayBackgroundColor }
+						textColor={ detectedOverlayColor }
+					/>
+				</>
+			) }
+		</>
+	);
+}
+
 function Navigation( {
 	attributes,
 	setAttributes,
@@ -330,17 +440,6 @@ function Navigation( {
 		},
 	} );
 
-	// Turn on contrast checker for web only since it's not supported on mobile yet.
-	const enableContrastChecking = Platform.OS === 'web';
-
-	const [ detectedBackgroundColor, setDetectedBackgroundColor ] = useState();
-	const [ detectedColor, setDetectedColor ] = useState();
-	const [
-		detectedOverlayBackgroundColor,
-		setDetectedOverlayBackgroundColor,
-	] = useState();
-	const [ detectedOverlayColor, setDetectedOverlayColor ] = useState();
-
 	const onSelectClassicMenu = async ( classicMenu ) => {
 		return convertClassicMenu( classicMenu.id, classicMenu.name, 'draft' );
 	};
@@ -412,40 +511,6 @@ function Navigation( {
 		handleUpdateMenu,
 	] );
 
-	useEffect( () => {
-		if ( ! enableContrastChecking ) {
-			return;
-		}
-		detectColors(
-			navRef.current,
-			setDetectedColor,
-			setDetectedBackgroundColor
-		);
-
-		const subMenuElement = navRef.current?.querySelector(
-			'[data-type="core/navigation-submenu"] [data-type="core/navigation-link"]'
-		);
-
-		if ( ! subMenuElement ) {
-			return;
-		}
-
-		// Only detect submenu overlay colors if they have previously been explicitly set.
-		// This avoids the contrast checker from reporting on inherited submenu colors and
-		// showing the contrast warning twice.
-		if ( overlayTextColor.color || overlayBackgroundColor.color ) {
-			detectColors(
-				subMenuElement,
-				setDetectedOverlayColor,
-				setDetectedOverlayBackgroundColor
-			);
-		}
-	}, [
-		enableContrastChecking,
-		overlayTextColor.color,
-		overlayBackgroundColor.color,
-	] );
-
 	useEffect( () => {
 		if ( ! isSelected && ! isInnerBlockSelected ) {
 			hideNavigationMenuPermissionsNotice();
@@ -518,7 +583,6 @@ function Navigation( {
 		`overlay-menu-preview`
 	);
 
-	const colorGradientSettings = useMultipleOriginColorsAndGradients();
 	const stylingInspectorControls = (
 		<>
 			<InspectorControls>
@@ -634,58 +698,25 @@ function Navigation( {
 					</PanelBody>
 				) }
 			</InspectorControls>
-			{ colorGradientSettings.hasColorsOrGradients && (
-				<InspectorControls group="color">
-					<ColorGradientSettingsDropdown
-						__experimentalIsRenderedInSidebar
-						settings={ [
-							{
-								colorValue: textColor.color,
-								label: __( 'Text' ),
-								onColorChange: setTextColor,
-								resetAllFilter: () => setTextColor(),
-							},
-							{
-								colorValue: backgroundColor.color,
-								label: __( 'Background' ),
-								onColorChange: setBackgroundColor,
-								resetAllFilter: () => setBackgroundColor(),
-							},
-							{
-								colorValue: overlayTextColor.color,
-								label: __( 'Submenu & overlay text' ),
-								onColorChange: setOverlayTextColor,
-								resetAllFilter: () => setOverlayTextColor(),
-							},
-							{
-								colorValue: overlayBackgroundColor.color,
-								label: __( 'Submenu & overlay background' ),
-								onColorChange: setOverlayBackgroundColor,
-								resetAllFilter: () =>
-									setOverlayBackgroundColor(),
-							},
-						] }
-						panelId={ clientId }
-						{ ...colorGradientSettings }
-						gradients={ [] }
-						disableCustomGradients
-					/>
-					{ enableContrastChecking && (
-						<>
-							<ContrastChecker
-								backgroundColor={ detectedBackgroundColor }
-								textColor={ detectedColor }
-							/>
-							<ContrastChecker
-								backgroundColor={
-									detectedOverlayBackgroundColor
-								}
-								textColor={ detectedOverlayColor }
-							/>
-						</>
-					) }
-				</InspectorControls>
-			) }
+			<InspectorControls group="color">
+				{ /*
+				 * Avoid useMultipleOriginColorsAndGradients and detectColors
+				 * on block mount. InspectorControls only mounts this component
+				 * when the block is selected.
+				 * */ }
+				<ColorTools
+					textColor={ textColor }
+					setTextColor={ setTextColor }
+					backgroundColor={ backgroundColor }
+					setBackgroundColor={ setBackgroundColor }
+					overlayTextColor={ overlayTextColor }
+					setOverlayTextColor={ setOverlayTextColor }
+					overlayBackgroundColor={ overlayBackgroundColor }
+					setOverlayBackgroundColor={ setOverlayBackgroundColor }
+					clientId={ clientId }
+					navRef={ navRef }
+				/>
+			</InspectorControls>
 		</>
 	);