Interacting Telemetry [2/n]: Hook up mutable AIConfig events to datad…

…og logger This builds upon the PR from https://github.com/lastmile-ai/gradio-workbook/pull/184 where we defined the `session_id_manual` param for each session. Now we're just attaching this to all events that have the ability to change the AIConfig (we already had the run command, add prompt, download and share buttons). Note that for all the text-based ones, these fire every time we do text input chnage, so it will disproportionally spam the loggers, but we can adjust sample rate for those as a P1 in the future if needed. I heard overall that storage is not a concern for us at the moment, so feel it's fine to do this ## Test Plan Go to the `aiconfig/python/src/aiconfig/editor/client` directory and run this command ``` rm -rf node_modules && yarn && yarn build ``` Then from the `aiconfig` top-level directory, run this ``` aiconfig_path=./cookbooks/Gradio/huggingface.aiconfig.json parsers_path=./cookbooks/Gradio/aiconfig_model_registry.py aiconfig edit --aiconfig-path=$aiconfig_path --server-port=8080 --server-mode=debug_servers --parsers-module-path=$parsers_path ``` Comment out the if-else blocks from https://github.com/lastmile-ai/aiconfig/blob/601eaf64b8b1af5d7172457da2433d6bfdad1d09/python/src/aiconfig/editor/client/src/LocalEditor.tsx#L59-L62 <img width="1089" alt="Screenshot 2024-02-12 at 22 43 29" src="https://github.com/lastmile-ai/aiconfig/assets/151060367/f64ba580-03d0-4318-a334-e58aebb66882">
lastmile-ai · Feb 13, 2024 · a4e5c74 · a4e5c74
1 parent 601eaf6
commit a4e5c74
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 21 deletions.
diff --git a/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx b/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx
@@ -278,6 +278,7 @@ function AIConfigEditorBase({
       };
 
       dispatch(action);
+      logEventHandler?.("UPDATE_PROMPT_INPUT");
 
       const onError = (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -313,7 +314,7 @@ function AIConfigEditorBase({
         onError(err);
       }
     },
-    [debouncedUpdatePrompt, dispatch, showNotification]
+    [debouncedUpdatePrompt, dispatch, logEventHandler, showNotification]
   );
 
   const onChangePromptName = useCallback(
@@ -349,19 +350,21 @@ function AIConfigEditorBase({
           // PromptName component maintains local state for the name to show in the UI
           // We cannot update client config state until the name is successfully set server-side
           // or else we could end up referencing a prompt name that is not set server-side
-          () =>
+          () => {
             dispatch({
               type: "UPDATE_PROMPT_NAME",
               id: promptId,
               name: newName,
-            }),
+            });
+            logEventHandler?.("UPDATE_PROMPT_NAME");
+          },
           onError
         );
       } catch (err: unknown) {
         onError(err);
       }
     },
-    [debouncedUpdatePrompt, showNotification]
+    [debouncedUpdatePrompt, logEventHandler, showNotification]
   );
 
   const updateModelCallback = callbacks?.updateModel;
@@ -402,6 +405,7 @@ function AIConfigEditorBase({
         id: promptId,
         modelSettings: newModelSettings,
       });
+      logEventHandler?.("UPDATE_PROMPT_MODEL_SETTINGS");
 
       const onError = (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -436,7 +440,7 @@ function AIConfigEditorBase({
         onError(err);
       }
     },
-    [debouncedUpdateModel, dispatch, showNotification]
+    [debouncedUpdateModel, dispatch, logEventHandler, showNotification]
   );
 
   const onUpdatePromptModel = useCallback(
@@ -452,6 +456,7 @@ function AIConfigEditorBase({
         id: promptId,
         modelName: newModel,
       });
+      logEventHandler?.("UPDATE_PROMPT_MODEL", { model: newModel });
 
       const onError = (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -479,7 +484,7 @@ function AIConfigEditorBase({
         onError(err);
       }
     },
-    [dispatch, debouncedUpdateModel, showNotification]
+    [dispatch, debouncedUpdateModel, logEventHandler, showNotification]
   );
 
   const setParametersCallback = callbacks?.setParameters;
@@ -516,6 +521,7 @@ function AIConfigEditorBase({
         type: "UPDATE_GLOBAL_PARAMETERS",
         parameters: newParameters,
       });
+      logEventHandler?.("UPDATE_GLOBAL_PARAMETERS");
 
       const onError = (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -536,7 +542,7 @@ function AIConfigEditorBase({
         onError(err);
       }
     },
-    [debouncedSetParameters, dispatch, showNotification]
+    [debouncedSetParameters, dispatch, logEventHandler, showNotification]
   );
 
   const onUpdatePromptParameters = useCallback(
@@ -552,6 +558,7 @@ function AIConfigEditorBase({
         id: promptId,
         parameters: newParameters,
       });
+      logEventHandler?.("UPDATE_PROMPT_PARAMETERS");
 
       const onError = (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -574,7 +581,7 @@ function AIConfigEditorBase({
         onError(err);
       }
     },
-    [debouncedSetParameters, dispatch, showNotification]
+    [debouncedSetParameters, dispatch, logEventHandler, showNotification]
   );
 
   const addPromptCallback = callbacks?.addPrompt;
@@ -648,6 +655,7 @@ function AIConfigEditorBase({
         type: "DELETE_PROMPT",
         id: promptId,
       });
+      logEventHandler?.("DELETE_PROMPT");
 
       try {
         const prompt = getPrompt(stateRef.current, promptId);
@@ -664,7 +672,7 @@ function AIConfigEditorBase({
         });
       }
     },
-    [deletePromptCallback, dispatch, showNotification]
+    [deletePromptCallback, dispatch, logEventHandler, showNotification]
   );
 
   const clearOutputsCallback = callbacks?.clearOutputs;
@@ -678,6 +686,8 @@ function AIConfigEditorBase({
     dispatch({
       type: "CLEAR_OUTPUTS",
     });
+    logEventHandler?.("CLEAR_OUTPUTS");
+
     try {
       await clearOutputsCallback();
     } catch (err: unknown) {
@@ -688,7 +698,7 @@ function AIConfigEditorBase({
         type: "error",
       });
     }
-  }, [clearOutputsCallback, dispatch, showNotification]);
+  }, [clearOutputsCallback, dispatch, logEventHandler, showNotification]);
 
   const runPromptCallback = callbacks?.runPrompt;
 
@@ -844,6 +854,7 @@ function AIConfigEditorBase({
         type: "SET_NAME",
         name,
       });
+      logEventHandler?.("SET_NAME");
 
       await debouncedSetName(name, (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -854,7 +865,7 @@ function AIConfigEditorBase({
         });
       });
     },
-    [debouncedSetName, showNotification]
+    [debouncedSetName, logEventHandler, showNotification]
   );
 
   const setDescriptionCallback = callbacks?.setConfigDescription;
@@ -887,6 +898,7 @@ function AIConfigEditorBase({
         type: "SET_DESCRIPTION",
         description,
       });
+      logEventHandler?.("SET_DESCRIPTION");
 
       await debouncedSetDescription(description, (err: unknown) => {
         const message = (err as RequestCallbackError).message ?? null;
@@ -897,7 +909,7 @@ function AIConfigEditorBase({
         });
       });
     },
-    [debouncedSetDescription, showNotification]
+    [debouncedSetDescription, logEventHandler, showNotification]
   );
 
   const getState = useCallback(() => stateRef.current, []);

diff --git a/python/src/aiconfig/editor/client/src/shared/types.ts b/python/src/aiconfig/editor/client/src/shared/types.ts
@@ -65,13 +65,23 @@ export function aiConfigToClientConfig(aiconfig: AIConfig): ClientAIConfig {
 
 export type LogEvent =
   | "ADD_PROMPT"
+  | "CLEAR_OUTPUTS"
+  | "DELETE_PROMPT"
   | "DOWNLOAD_BUTTON_CLICKED"
   | "RUN_PROMPT_CANCELED"
   | "RUN_PROMPT_ERROR"
   | "RUN_PROMPT_START"
   | "RUN_PROMPT_SUCCESS"
   | "SAVE_BUTTON_CLICKED"
-  | "SHARE_BUTTON_CLICKED";
+  | "SET_DESCRIPTION"
+  | "SET_NAME"
+  | "SHARE_BUTTON_CLICKED"
+  | "UPDATE_GLOBAL_PARAMETERS"
+  | "UPDATE_PROMPT_INPUT"
+  | "UPDATE_PROMPT_MODEL"
+  | "UPDATE_PROMPT_MODEL_SETTINGS"
+  | "UPDATE_PROMPT_NAME"
+  | "UPDATE_PROMPT_PARAMETERS";
 
 // TODO: schematize this
 export type LogEventData = JSONObject;

diff --git a/python/src/aiconfig/editor/server/static/asset-manifest.json b/python/src/aiconfig/editor/server/static/asset-manifest.json
@@ -1,10 +1,10 @@
 {
   "files": {
-    "main.js": "/static/js/main.e93da0c9.js",
+    "main.js": "/static/js/main.f838c843.js",
     "index.html": "/index.html",
-    "main.e93da0c9.js.map": "/static/js/main.e93da0c9.js.map"
+    "main.f838c843.js.map": "/static/js/main.f838c843.js.map"
   },
   "entrypoints": [
-    "static/js/main.e93da0c9.js"
+    "static/js/main.f838c843.js"
   ]
 }
diff --git a/python/src/aiconfig/editor/server/static/index.html b/python/src/aiconfig/editor/server/static/index.html
@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>AIConfig Editor</title><script defer="defer" src="/static/js/main.e93da0c9.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>AIConfig Editor</title><script defer="defer" src="/static/js/main.f838c843.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
diff --git a/.../server/static/static/js/main.e93da0c9.js → .../server/static/static/js/main.f838c843.js b/.../server/static/static/js/main.e93da0c9.js → .../server/static/static/js/main.f838c843.js
diff --git a/...ic/static/js/main.e93da0c9.js.LICENSE.txt → ...ic/static/js/main.f838c843.js.LICENSE.txt b/...ic/static/js/main.e93da0c9.js.LICENSE.txt → ...ic/static/js/main.f838c843.js.LICENSE.txt
diff --git a/...ver/static/static/js/main.e93da0c9.js.map → ...ver/static/static/js/main.f838c843.js.map b/...ver/static/static/js/main.e93da0c9.js.map → ...ver/static/static/js/main.f838c843.js.map