diff --git a/alert-policies/langchain-vectordb/APIResponseTime.yml b/alert-policies/langchain-vectordb/APIResponseTime.yml new file mode 100644 index 0000000000..c279327bf2 --- /dev/null +++ b/alert-policies/langchain-vectordb/APIResponseTime.yml @@ -0,0 +1,32 @@ +# Name of the alert +name: API Response Time(seconds) + +# Description and details +description: |+ + This alert is triggered when the response time exceeds 2 seconds for 1 minute. + +# Type of alert +type: STATIC + +# NRQL query +nrql: + query: "FROM LlmVectorSearch SELECT latest(response_time) WHERE applicationName IN ('LangChain observability trace') TIMESERIES" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 2 + # Time in seconds; 120 - 3600 + thresholdDuration: 120 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 diff --git a/alert-policies/langchain-vectordb/ZeroDocumentsRetrieved.yml b/alert-policies/langchain-vectordb/ZeroDocumentsRetrieved.yml new file mode 100644 index 0000000000..c7088edb3e --- /dev/null +++ b/alert-policies/langchain-vectordb/ZeroDocumentsRetrieved.yml @@ -0,0 +1,41 @@ +# Name of the alert +name: Zero Documents Retrieved + +# Description and details +description: |+ + This alert is triggered when more than 20% of the searches return zero documents for 1 minute. + +# Type of alert +type: STATIC + +# NRQL query +nrql: + query: "SELECT filter(count(document_count), WHERE document_count=0) / count(document_count) * 100 from LlmVectorSearch WHERE applicationName IN ('LangChain observability trace')" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 50 + # Time in seconds; 120 - 3600 + thresholdDuration: 120 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + - priority: WARNING + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 20 + # Time in seconds; 120 - 3600 + thresholdDuration: 120 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 diff --git a/dashboards/langchain-vectordb/langchain-vectordb-01.png b/dashboards/langchain-vectordb/langchain-vectordb-01.png new file mode 100644 index 0000000000..951d37d1e2 Binary files /dev/null and b/dashboards/langchain-vectordb/langchain-vectordb-01.png differ diff --git a/dashboards/langchain-vectordb/langchain-vectordb-02.png b/dashboards/langchain-vectordb/langchain-vectordb-02.png new file mode 100644 index 0000000000..84c3cab6d4 Binary files /dev/null and b/dashboards/langchain-vectordb/langchain-vectordb-02.png differ diff --git a/dashboards/langchain-vectordb/langchain-vectordb-03.png b/dashboards/langchain-vectordb/langchain-vectordb-03.png new file mode 100644 index 0000000000..cbf6ba4956 Binary files /dev/null and b/dashboards/langchain-vectordb/langchain-vectordb-03.png differ diff --git a/dashboards/langchain-vectordb/langchain-vectordb-04.png b/dashboards/langchain-vectordb/langchain-vectordb-04.png new file mode 100644 index 0000000000..f353779e6f Binary files /dev/null and b/dashboards/langchain-vectordb/langchain-vectordb-04.png differ diff --git a/dashboards/langchain-vectordb/langchain-vectordb-05.png b/dashboards/langchain-vectordb/langchain-vectordb-05.png new file mode 100644 index 0000000000..76abaaa84c Binary files /dev/null and b/dashboards/langchain-vectordb/langchain-vectordb-05.png differ diff --git a/dashboards/langchain-vectordb/langchain-vectordb.json b/dashboards/langchain-vectordb/langchain-vectordb.json new file mode 100644 index 0000000000..9430a3fde4 --- /dev/null +++ b/dashboards/langchain-vectordb/langchain-vectordb.json @@ -0,0 +1,792 @@ +{ + "name": "LangChain", + "description": null, + "pages": [ + { + "name": "LangChain vector stores", + "description": null, + "widgets": [ + { + "title": "Response Time (seconds)", + "layout": { + "column": 1, + "row": 1, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "FROM LlmVectorSearch SELECT latest(response_time) WHERE applicationName IN ({{applicationName}}) TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Searches by provider", + "layout": { + "column": 5, + "row": 1, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "FROM LlmVectorSearch SELECT count(*) WHERE applicationName IN ({{applicationName}}) FACET provider" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Average document retrieved", + "layout": { + "column": 9, + "row": 1, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT average(document_count) from LlmVectorSearch WHERE applicationName IN ({{applicationName}})" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "List of queries ", + "layout": { + "column": 1, + "row": 4, + "width": 3, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(query) FROM LlmVectorSearch WHERE applicationName IN ({{applicationName}}) FACET query " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "List of search id's", + "layout": { + "column": 4, + "row": 4, + "width": 3, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) from LlmVectorSearch WHERE applicationName IN ({{applicationName}}) FACET search_id " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Page source and content", + "layout": { + "column": 7, + "row": 4, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT result_rank, document_metadata_source, document_page_content FROM LlmVectorSearchResult" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 7, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# Popular content\nThe following metrics will help you understand what content is popular in your system, so that you can improve your application, its content, and the indexing of that content." + } + }, + { + "title": "Top metadata source", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) FROM LlmVectorSearchResult FACET document_metadata_source " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Top page content", + "layout": { + "column": 5, + "row": 8, + "width": 8, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) FROM LlmVectorSearchResult FACET document_page_content" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + } + ] + }, + { + "name": "LangChain", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# LLM Completion\nThe following metrics will help you identify trends, compare models and optimize your LLM." + } + }, + { + "title": "Average tokens", + "layout": { + "column": 1, + "row": 2, + "width": 2, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT average(prompt_tokens) as 'Prompt tokens',average(completion_tokens) as 'Completion tokens',average(total_tokens) as 'Total tokens' from LlmCompletion " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 3, + "row": 2, + "width": 2, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmCompletion " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 5, + "row": 2, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmCompletion TIMESERIES " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Models", + "layout": { + "column": 9, + "row": 2, + "width": 4, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.pie" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) from LlmCompletion facet model_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds) by model name", + "layout": { + "column": 5, + "row": 4, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT median(duration.ms/1000) from LlmCompletion facet model_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "LLM completion's details", + "layout": { + "column": 1, + "row": 6, + "width": 12, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT model_name,messages,response from LlmCompletion " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 10, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# LLM Chains\nThe following metrics will help you to get a better understanding of how your LLM chains are performing and to identify any potential problems." + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 1, + "row": 11, + "width": 2, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmChain" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 3, + "row": 11, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmChain TIMESERIES " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Chain's details", + "layout": { + "column": 7, + "row": 11, + "width": 6, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT chat_history,input,outputs,guid,trace.id ,parent.id from LlmChain since 3 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Chains triggered", + "layout": { + "column": 3, + "row": 13, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) from LlmChain TIMESERIES " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 15, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# LLM Tools\nThe following metrics helps you to understand how your LLM tools are performing and to identify any potential problems.\n " + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 1, + "row": 16, + "width": 2, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmTool " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds)", + "layout": { + "column": 3, + "row": 16, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms/1000),95) as 'seconds',percentile((duration.ms/1000),99) as 'seconds',percentile((duration.ms/1000),50) as 'seconds' from LlmTool TIMESERIES " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Tool details", + "layout": { + "column": 7, + "row": 16, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT tool_name,tool_invocation_counter, tool_input,tool_output from LlmTool " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds) by tool name ", + "layout": { + "column": 1, + "row": 19, + "width": 2, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT average(duration.ms/1000) as 'seconds' from LlmTool facet tool_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Duration (seconds) by tool name ", + "layout": { + "column": 3, + "row": 19, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT percentile((duration.ms / 1000),50) as 'seconds' from LlmTool facet tool_name TIMESERIES " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Top tool names", + "layout": { + "column": 9, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "SELECT count(*) from LlmTool facet tool_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + } + ] + } + ], + "variables": [ + { + "name": "applicationName", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + }, + { + "value": { + "string": "LangChain observability trace" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "FROM LlmVectorSearch SELECT uniques(applicationName) WHERE applicationName IS NOT NULL since 1 month ago" + }, + "title": "Select Your Application Name", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + } + ] +} \ No newline at end of file diff --git a/data-sources/langchain-vectordb/config.yml b/data-sources/langchain-vectordb/config.yml new file mode 100644 index 0000000000..8ad1c143ed --- /dev/null +++ b/data-sources/langchain-vectordb/config.yml @@ -0,0 +1,23 @@ +id: langchain-vectordb +displayName: LangChain Vector Database +description: | + Implement monitoring and instrumentation for your LangChain Vector Database and ensure that your vector database is integrated into New Relic for effective performance analysis and insights. +install: + primary: + nerdlet: + nerdletId: marketplace.install-data-source + nerdletState: + dataSourceId: langchain-vectordb + requiresAccount: false +icon: logo.png +keywords: + - langchain vector database + - langchain ai + - lang + - chain + - ai + - lang chain + - large language model + - natural language processing + - machine learning + - artificial intelligence \ No newline at end of file diff --git a/data-sources/langchain-vectordb/logo.png b/data-sources/langchain-vectordb/logo.png new file mode 100644 index 0000000000..240e64b569 Binary files /dev/null and b/data-sources/langchain-vectordb/logo.png differ diff --git a/quickstarts/langchain-vectordb/config.yml b/quickstarts/langchain-vectordb/config.yml new file mode 100644 index 0000000000..563b604760 --- /dev/null +++ b/quickstarts/langchain-vectordb/config.yml @@ -0,0 +1,46 @@ +slug: langchain-vectordb +description: | + ## Why should you monitor your usage of Vector Stores? + + Monitor your vector searches to get visibility on what you send to your vector store, responses retrieved from them, latency, usage and errors. + + ### Track the query performance of your Vector DB + Track the behavior of your vector stores. Monitor the latency, queries, the number of documents retrieved, and the content of the documents so that you can evaluate their relevance. + + ### Track your app: + By tracking key metrics like latency, throughput, error rates, and input & output, you can gain insights into your app's performance and identify areas of improvement. + + ### What’s included in this quickstart? + + New Relic LangChain monitoring quickstart provides a variety of pre-built dashboards, which will help you gain insights into the health and performance of your LangChain app. These reports include: + + - Vector searches + - Alerts for errors, search per vector store, and response time + - Identify popular queries, sources, and content + +summary: | + Monitor your Vector search's performance and quality with New Relic LangChain Vector Stores quickstart. +icon: logo.png +level: New Relic +authors: + - New Relic +title: LangChain Vector Stores +documentation: + - name: LangChain Vector Database integration documentation + description: | + Implement monitoring and instrumentation for your Vector store, and ensure that your observability data is integrated into New Relic for effective performance analysis and insights. + url: https://github.com/newrelic/nr-openai-observability +dataSourceIds: + - langchain-vectordb +keywords: + - langchain + - vector store + - vector search + - vectordb + - vector db + - mlops + - NR1_addData +dashboards: + - langchain-vectordb +alertPolicies: + - langchain-vectordb \ No newline at end of file diff --git a/quickstarts/langchain-vectordb/logo.png b/quickstarts/langchain-vectordb/logo.png new file mode 100644 index 0000000000..240e64b569 Binary files /dev/null and b/quickstarts/langchain-vectordb/logo.png differ