diff --git a/src/plugins/visualizations/public/embeddable/state.ts b/src/plugins/visualizations/public/embeddable/state.ts
index 52641703e04b6..c94e7a90eeff1 100644
--- a/src/plugins/visualizations/public/embeddable/state.ts
+++ b/src/plugins/visualizations/public/embeddable/state.ts
@@ -30,13 +30,11 @@ import {
 import { getSavedVisualization } from '../utils/saved_visualize_utils';
 import type { SerializedVis } from '../vis';
 import {
-  isVisualizeSavedObjectState,
   VisualizeSavedObjectInputState,
   VisualizeSerializedState,
   VisualizeRuntimeState,
   VisualizeSavedVisInputState,
   ExtraSavedObjectProperties,
-  isVisualizeRuntimeState,
 } from './types';
 
 export const deserializeState = async (
@@ -49,15 +47,24 @@ export const deserializeState = async (
       },
     } as VisualizeRuntimeState;
   let serializedState = cloneDeep(state.rawState);
-  if (isVisualizeSavedObjectState(serializedState)) {
-    serializedState = await deserializeSavedObjectState(serializedState);
-  } else if (isVisualizeRuntimeState(serializedState)) {
+  if ((serializedState as VisualizeSavedObjectInputState).savedObjectId) {
+    serializedState = await deserializeSavedObjectState(
+      serializedState as VisualizeSavedObjectInputState
+    );
+  } else if ((serializedState as VisualizeRuntimeState).serializedVis) {
+    // TODO remove once embeddable only exposes SerializedState
+    // Canvas passes incoming embeddable state in getSerializedStateForChild
+    // without this early return, serializedVis gets replaced in deserializeSavedVisState
+    // and breaks adding a new by-value embeddable in Canvas
     return serializedState as VisualizeRuntimeState;
   }
 
   const references: Reference[] = state.references ?? [];
 
-  const deserializedSavedVis = deserializeSavedVisState(serializedState, references);
+  const deserializedSavedVis = deserializeSavedVisState(
+    serializedState as VisualizeSavedVisInputState,
+    references
+  );
 
   return {
     ...serializedState,
diff --git a/src/plugins/visualizations/public/embeddable/types.ts b/src/plugins/visualizations/public/embeddable/types.ts
index f4215d923e1d5..1c535b4875dcc 100644
--- a/src/plugins/visualizations/public/embeddable/types.ts
+++ b/src/plugins/visualizations/public/embeddable/types.ts
@@ -69,26 +69,6 @@ export type VisualizeOutputState = VisualizeSavedVisInputState &
   Required<Omit<SerializedTitles, 'hidePanelTitles'>> &
   ExtraSavedObjectProperties;
 
-export const isVisualizeSavedObjectState = (
-  state: unknown
-): state is VisualizeSavedObjectInputState => {
-  return (
-    typeof state !== 'undefined' &&
-    (state as VisualizeSavedObjectInputState).savedObjectId !== undefined &&
-    !!(state as VisualizeSavedObjectInputState).savedObjectId &&
-    !('savedVis' in (state as VisualizeSavedObjectInputState)) &&
-    !('serializedVis' in (state as VisualizeSavedObjectInputState))
-  );
-};
-
-export const isVisualizeRuntimeState = (state: unknown): state is VisualizeRuntimeState => {
-  return (
-    !isVisualizeSavedObjectState(state) &&
-    !('savedVis' in (state as VisualizeRuntimeState)) &&
-    (state as VisualizeRuntimeState).serializedVis !== undefined
-  );
-};
-
 export type VisualizeApi = Partial<HasEditCapabilities> &
   PublishesDataViews &
   PublishesDataLoading &
diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx b/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx
index 7b48521265d6f..9ac0674c2f30d 100644
--- a/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx
+++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx
@@ -54,7 +54,6 @@ import {
   VisualizeOutputState,
   VisualizeRuntimeState,
   VisualizeSerializedState,
-  isVisualizeSavedObjectState,
 } from './types';
 import { initializeEditApi } from './initialize_edit_api';
 
@@ -67,15 +66,11 @@ export const getVisualizeEmbeddableFactory: (deps: {
 }) => ({
   type: VISUALIZE_EMBEDDABLE_TYPE,
   deserializeState,
-  buildEmbeddable: async (initialState: unknown, buildApi, uuid, parentApi) => {
-    // Handle state transfer from legacy visualize editor, which uses the legacy visualize embeddable and doesn't
-    // produce a snapshot state. If buildEmbeddable is passed only a savedObjectId in the state, this means deserializeState
-    // was never run, and it needs to be invoked manually
-    const state = isVisualizeSavedObjectState(initialState)
-      ? await deserializeState({
-          rawState: initialState,
-        })
-      : (initialState as VisualizeRuntimeState);
+  buildEmbeddable: async (initialState, buildApi, uuid, parentApi) => {
+    const state = {
+      ...initialState,
+      linkedToLibrary: Boolean(initialState.savedObjectId),
+    };
 
     // Initialize dynamic actions
     const dynamicActionsApi = embeddableEnhancedStart?.initializeReactEmbeddableDynamicActions(
@@ -312,7 +307,13 @@ export const getVisualizeEmbeddableFactory: (deps: {
           },
         ],
         savedObjectProperties: getUnchangingComparator(),
-        linkedToLibrary: [linkedToLibrary$, (value) => linkedToLibrary$.next(value)],
+        linkedToLibrary: [
+          linkedToLibrary$,
+          (value) => linkedToLibrary$.next(value),
+          (a, b) => {
+            return a === undefined || b === undefined ? true : a === b;
+          },
+        ],
       }
     );
 
diff --git a/src/plugins/visualizations/public/plugin.ts b/src/plugins/visualizations/public/plugin.ts
index fcb3dc5137161..9cf4e93ead188 100644
--- a/src/plugins/visualizations/public/plugin.ts
+++ b/src/plugins/visualizations/public/plugin.ts
@@ -125,7 +125,7 @@ import {
   VisualizationSavedObjectAttributes,
 } from '../common/content_management';
 import { AddAggVisualizationPanelAction } from './actions/add_agg_vis_action';
-import type { VisualizeSerializedState } from './embeddable/types';
+import type { VisualizeRuntimeState } from './embeddable/types';
 import { getVisualizeEmbeddableFactoryLazy } from './embeddable';
 
 /**
@@ -412,10 +412,15 @@ export class VisualizationsPlugin
       return getVisualizeEmbeddableFactory({ embeddableStart, embeddableEnhancedStart });
     });
     embeddable.registerReactEmbeddableSavedObject<VisualizationSavedObjectAttributes>({
-      onAdd: (container, savedObject) => {
-        container.addNewPanel<VisualizeSerializedState>({
+      onAdd: async (container, savedObject) => {
+        const { deserializeState } = await import('./embeddable/state');
+        const initialState = await deserializeState({
+          rawState: { savedObjectId: savedObject.id },
+          references: savedObject.references,
+        });
+        container.addNewPanel<VisualizeRuntimeState>({
           panelType: VISUALIZE_EMBEDDABLE_TYPE,
-          initialState: { savedObjectId: savedObject.id },
+          initialState,
         });
       },
       embeddableType: VISUALIZE_EMBEDDABLE_TYPE,
diff --git a/src/plugins/visualizations/public/visualize_app/utils/get_top_nav_config.tsx b/src/plugins/visualizations/public/visualize_app/utils/get_top_nav_config.tsx
index 3dd181a85e4e1..245791a7460f5 100644
--- a/src/plugins/visualizations/public/visualize_app/utils/get_top_nav_config.tsx
+++ b/src/plugins/visualizations/public/visualize_app/utils/get_top_nav_config.tsx
@@ -187,7 +187,10 @@ export const getTopNavConfig = (
             stateTransfer.navigateToWithEmbeddablePackage(app, {
               state: {
                 type: VISUALIZE_EMBEDDABLE_TYPE,
-                input: { savedObjectId: id },
+                input: {
+                  serializedVis: vis.serialize(),
+                  savedObjectId: id,
+                },
                 embeddableId: saveOptions.copyOnSave ? undefined : embeddableId,
                 searchSessionId: data.search.session.getSessionId(),
               },