From 5d2decc70104fe3cf1e9f115ea8d28572aa04264 Mon Sep 17 00:00:00 2001 From: "Ankush Pala ankush@lastmileai.dev" <> Date: Fri, 12 Jan 2024 19:28:36 -0500 Subject: [PATCH] [editor][client] Enable Telemetry based on User Config settings This diff builds on top of @rholinshead's starting point for telemetry diff. Most of that looked good to me so I didn't touch it beside @rossdanlm 's . - Building off of #869, if `allow_usage_data_sharing` is set to True, initialize Datadog Browser logging with a session ID. - Disabled telemetry in dev mode as per @rholinshead 's comment that the hot reload spams the logs with something like "datadog logging already initialized" - Moved the initialization logic away from`index.tsx` into `editor.tsx` so that it can be configurable. ## Testplan: 1. yarn build 2. run in "prod" mode 3. Edit AIconfig 4. Hit Save button -> validate datadog log sent repeat with aiconfig.rc set with false logging |Screenshot 2024-01-12 at 7 24 33 PM | Screenshot 2024-01-12 at 7 23 51 PM | | ------------- | ------------- | | Screenshot 2024-01-12 at 7 26 35 PM | -> No logs | I tried taking a video but datadog doesn't immediately update with logs which made the video too long to upload --- .../src/aiconfig/editor/client/package.json | 4 +-- .../editor/client/src/LocalEditor.tsx | 31 ++++++++++++++----- .../client/src/components/AIConfigEditor.tsx | 12 +++---- .../src/aiconfig/editor/client/src/index.tsx | 11 ------- .../editor/client/src/shared/types.ts | 1 + .../aiconfig/editor/client/src/utils/api.ts | 1 + 6 files changed, 34 insertions(+), 26 deletions(-) diff --git a/python/src/aiconfig/editor/client/package.json b/python/src/aiconfig/editor/client/package.json index fa843cbd3..ef9aa9696 100644 --- a/python/src/aiconfig/editor/client/package.json +++ b/python/src/aiconfig/editor/client/package.json @@ -23,7 +23,7 @@ ] }, "dependencies": { - "@datadog/browser-logs": "^5.6.0", + "@datadog/browser-logs": "^5.7.0", "@emotion/react": "^11.11.1", "@mantine/carousel": "^6.0.7", "@mantine/core": "^6.0.7", @@ -64,4 +64,4 @@ "eslint-plugin-react-hooks": "^4.6.0", "typescript": "^5" } -} \ No newline at end of file +} diff --git a/python/src/aiconfig/editor/client/src/LocalEditor.tsx b/python/src/aiconfig/editor/client/src/LocalEditor.tsx index 840019e10..1e9d3ebc1 100644 --- a/python/src/aiconfig/editor/client/src/LocalEditor.tsx +++ b/python/src/aiconfig/editor/client/src/LocalEditor.tsx @@ -5,13 +5,7 @@ import AIConfigEditor, { RunPromptStreamErrorEvent, } from "./components/AIConfigEditor"; import { Flex, Loader, MantineProvider, Image } from "@mantine/core"; -import { - AIConfig, - InferenceSettings, - JSONObject, - Output, - Prompt, -} from "aiconfig"; +import { AIConfig, InferenceSettings, JSONObject, Output, Prompt } from "aiconfig"; import { useCallback, useEffect, useMemo, useState } from "react"; import { ufetch } from "ufetch"; import { ROUTE_TABLE } from "./utils/api"; @@ -31,6 +25,29 @@ export default function Editor() { loadConfig(); }, [loadConfig]); + const setupTelemetryIfAllowed = useCallback(async () => { + const res = await ufetch.get(ROUTE_TABLE.GET_AICONFIGRC, {}); // change this to the endpoint used to get the config + + const enableTelemetry = res.allow_usage_data_sharing; + const isDev = (process.env.NODE_ENV ?? "development") === "development"; + + // disable telemetry in dev mode because hot reload will spam the logs. + if (enableTelemetry && !isDev) { + datadogLogs.init({ + clientToken: "pub356987caf022337989e492681d1944a8", + env: process.env.NODE_ENV ?? "development", + service: "aiconfig-editor", + site: "us5.datadoghq.com", + forwardErrorsToLogs: true, + sessionSampleRate: 100, + }); + } + }, []); + + useEffect(() => { + setupTelemetryIfAllowed(); + }, [setupTelemetryIfAllowed]); + const save = useCallback(async (aiconfig: AIConfig) => { const res = await ufetch.post(ROUTE_TABLE.SAVE, { // path: file path, diff --git a/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx b/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx index 0588cb8ed..b410db911 100644 --- a/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx +++ b/python/src/aiconfig/editor/client/src/components/AIConfigEditor.tsx @@ -174,7 +174,7 @@ export default function EditorContainer({ const stateRef = useRef(aiconfigState); stateRef.current = aiconfigState; - const logEvent = callbacks.logEvent; + const logEventCallback = callbacks.logEvent; const saveCallback = callbacks.save; const onSave = useCallback(async () => { @@ -524,7 +524,7 @@ export default function EditorContainer({ }; dispatch(action); - logEvent?.("ADD_PROMPT", { model, promptIndex }); + logEventCallback?.("ADD_PROMPT", { model, promptIndex }); try { const serverConfigRes = await addPromptCallback( @@ -546,7 +546,7 @@ export default function EditorContainer({ }); } }, - [addPromptCallback, logEvent] + [addPromptCallback, logEventCallback] ); const deletePromptCallback = callbacks.deletePrompt; @@ -777,9 +777,9 @@ export default function EditorContainer({ const contextValue = useMemo( () => ({ getState, - logEvent, + logEvent: logEventCallback, }), - [getState, logEvent] + [getState, logEventCallback] ); const isDirty = aiconfigState._ui.isDirty !== false; @@ -887,7 +887,7 @@ export default function EditorContainer({ loading={isSaving} onClick={() => { onSave(); - logEvent?.("SAVE_BUTTON_CLICKED"); + logEventCallback?.("SAVE_BUTTON_CLICKED"); }} disabled={!isDirty} size="xs" diff --git a/python/src/aiconfig/editor/client/src/index.tsx b/python/src/aiconfig/editor/client/src/index.tsx index c7fa1681d..fccc9ab4e 100644 --- a/python/src/aiconfig/editor/client/src/index.tsx +++ b/python/src/aiconfig/editor/client/src/index.tsx @@ -2,17 +2,6 @@ import React from "react"; import ReactDOM from "react-dom/client"; import LocalEditor from "./LocalEditor"; -import { datadogLogs } from "@datadog/browser-logs"; - -datadogLogs.init({ - clientToken: "pub356987caf022337989e492681d1944a8", - env: process.env.NODE_ENV ?? "development", - service: "aiconfig-editor", - site: "us5.datadoghq.com", - forwardErrorsToLogs: true, - sessionSampleRate: 100, -}); - const root = ReactDOM.createRoot( document.getElementById("root") as HTMLElement ); diff --git a/python/src/aiconfig/editor/client/src/shared/types.ts b/python/src/aiconfig/editor/client/src/shared/types.ts index 012cdc111..44b9d16be 100644 --- a/python/src/aiconfig/editor/client/src/shared/types.ts +++ b/python/src/aiconfig/editor/client/src/shared/types.ts @@ -63,4 +63,5 @@ export function aiConfigToClientConfig(aiconfig: AIConfig): ClientAIConfig { } export type LogEvent = "ADD_PROMPT" | "SAVE_BUTTON_CLICKED"; +// TODO: schematize this export type LogEventData = JSONObject; diff --git a/python/src/aiconfig/editor/client/src/utils/api.ts b/python/src/aiconfig/editor/client/src/utils/api.ts index 9ba4c82f6..a98f2f200 100644 --- a/python/src/aiconfig/editor/client/src/utils/api.ts +++ b/python/src/aiconfig/editor/client/src/utils/api.ts @@ -14,6 +14,7 @@ export const ROUTE_TABLE = { CANCEL: urlJoin(API_ENDPOINT, "/cancel"), CLEAR_OUTPUTS: urlJoin(API_ENDPOINT, "/clear_outputs"), DELETE_PROMPT: urlJoin(API_ENDPOINT, "/delete_prompt"), + GET_AICONFIGRC: urlJoin(API_ENDPOINT, "/get_aiconfigrc"), SAVE: urlJoin(API_ENDPOINT, "/save"), SET_DESCRIPTION: urlJoin(API_ENDPOINT, "/set_description"), SERVER_STATUS: urlJoin(API_ENDPOINT, "/server_status"),