Merge pull request #2243 from glavanr/galva/add-fivetran-quickstart

Add fivetran quickstart

alert-policies/fivetran/ConnectionFailure.yml

@@ -0,0 +1,28 @@
+name: Connection failure
+
+description: |+
+ This alert is triggered in the event of a connection failure, indicating the failure of any Fivetran jobs.
+  
+type: STATIC
+
+nrql:
+  query: "SELECT count(*) from Log where event = 'connection_failure'"
+
+# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE)
+valueFunction: SINGLE_VALUE 
+
+# List of Critical and Warning thresholds for the condition
+terms:
+  - priority: CRITICAL
+    # Operator used to compare against the threshold.
+    operator: EQUALS
+    # Value that triggers a violation; float value
+    threshold: 1
+    # Time in seconds; 120 - 3600
+    thresholdDuration: 300
+    # How many data points must be in violation for the duration
+    thresholdOccurrences: AT_LEAST_ONCE
+
+# Duration after which a violation automatically closes
+# Time in seconds; 300 - 2592000 (Default: 86400 [1 day])
+violationTimeLimitSeconds: 2592000

alert-policies/fivetran/FailureWithTask.yml

@@ -0,0 +1,28 @@
+name: Task failure
+
+description: |+
+  This alert is triggered in the event of a connection failure, indicating the failure of any Fivetran task.
+  
+type: STATIC
+
+nrql:
+  query: "SELECT count(*) from Log where event = 'sync_end' and data.status ='FAILURE_WITH_TASK'"
+
+# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE)
+valueFunction: SINGLE_VALUE 
+
+# List of Critical and Warning thresholds for the condition
+terms:
+  - priority: CRITICAL
+    # Operator used to compare against the threshold.
+    operator: EQUALS
+    # Value that triggers a violation; float value
+    threshold: 1
+    # Time in seconds; 120 - 3600
+    thresholdDuration: 300
+    # How many data points must be in violation for the duration
+    thresholdOccurrences: AT_LEAST_ONCE
+
+# Duration after which a violation automatically closes
+# Time in seconds; 300 - 2592000 (Default: 86400 [1 day])
+violationTimeLimitSeconds: 2592000

dashboards/fivetran/fivetran-connector-monitoring.json

@@ -0,0 +1,131 @@
+{
+  "name": "Fivetran Monitoring Quick Start",
+  "description": null,
+  "pages": [
+    {
+      "name": "Fivetran Monitoring",
+      "description": "Monitors Fivetran connections here: https://fivetran.com/dashboard/connectors",
+      "widgets": [
+        {
+          "title": "Fivetran Overview",
+          "layout": {
+            "column": 1,
+            "row": 1,
+            "width": 4,
+            "height": 6
+          },
+          "linkedEntityGuids": null,
+          "visualization": {
+            "id": "viz.markdown"
+          },
+          "rawConfiguration": {
+            "text": "This dashboard is used to track Fivetran sync and connection failures."
+          }
+        },
+        {
+          "title": "Fivetran Successful Sync History",
+          "layout": {
+            "column": 5,
+            "row": 1,
+            "width": 4,
+            "height": 3
+          },
+          "linkedEntityGuids": null,
+          "visualization": {
+            "id": "viz.table"
+          },
+          "rawConfiguration": {
+            "facet": {
+              "showOtherSeries": true
+            },
+            "nrqlQueries": [
+              {
+                "accountIds": [],
+                "query": "SELECT connector_name, timestamp, data.status from Log where data.status = 'SUCCESSFUL' and event ='sync_end' SINCE 1 day ago LIMIT MAX ORDER BY timestamp DESC"
+              }
+            ],
+            "platformOptions": {
+              "ignoreTimeRange": false
+            }
+          }
+        },
+        {
+          "title": "Fivetran Failure Sync",
+          "layout": {
+            "column": 9,
+            "row": 1,
+            "width": 4,
+            "height": 3
+          },
+          "linkedEntityGuids": null,
+          "visualization": {
+            "id": "viz.table"
+          },
+          "rawConfiguration": {
+            "facet": {
+              "showOtherSeries": true
+            },
+            "nrqlQueries": [
+              {
+                "accountIds": [],
+                "query": "SELECT connector_name, timestamp, data.status from Log where data.status in ('FAILURE_WITH_TASK', 'FAILURE') and event ='sync_end' SINCE 1 week ago"
+              }
+            ],
+            "platformOptions": {
+              "ignoreTimeRange": false
+            }
+          }
+        },
+        {
+          "title": "Fivetran Topics Last Successful Sync",
+          "layout": {
+            "column": 5,
+            "row": 4,
+            "width": 4,
+            "height": 3
+          },
+          "linkedEntityGuids": null,
+          "visualization": {
+            "id": "viz.table"
+          },
+          "rawConfiguration": {
+            "facet": {
+              "showOtherSeries": true
+            },
+            "nrqlQueries": [
+              {
+                "accountIds": [],
+                "query": "SELECT latest(timestamp), latest(data.status) from Log where data.status = 'SUCCESSFUL' and event ='sync_end' SINCE 1 week ago LIMIT MAX FACET connector_name ORDER BY timestamp DESC"
+              }
+            ],
+            "platformOptions": {
+              "ignoreTimeRange": false
+            }
+          }
+        },
+        {
+          "title": "Fivetran Connection Failures",
+          "layout": {
+            "column": 9,
+            "row": 4,
+            "width": 4,
+            "height": 3
+          },
+          "linkedEntityGuids": null,
+          "visualization": {
+            "id": "logger.log-table-widget"
+          },
+          "rawConfiguration": {
+            "nrqlQueries": [
+              {
+                "accountIds": [],
+                "query": "SELECT connector_name, timestamp,data.status from Log where event = 'connection_failure' SINCE 1 week ago"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "variables": []
+}

data-sources/fivetran/config.yml

@@ -0,0 +1,16 @@
+id: fivetran
+displayName: Fivetran
+description: |
+  Fivetran is a cloud-based data integration platform that helps organizations streamline the process of extracting, loading, and transforming (ELT) data from various sources into a centralized data warehouse.
+icon: fivetran-logo.png
+install:
+  primary:
+    link:
+      url: https://fivetran.com/docs/logs/external-logs/new-relic/setup-guide
+keywords:
+  - data pipeline
+  - elt
+  - fivetran
+categoryTerms:
+  - newrelic partner
+  - logs

quickstarts/fivetran/config.yml

@@ -0,0 +1,32 @@
+slug: fivetran
+title: Fivetran Destination Monitoring
+
+description: |
+  Monitor the status of your Fivetran Connectors to gain visibility on broken connectors.
+
+summary: |
+  Monitor Fivetran sync history and detect broken connectors.
+
+level: Community
+
+authors:
+  - Michael Young
+
+keywords:
+  - fivetran
+  - connector
+  - pipeline
+icon: logo.jpeg
+# Reference to alert policies to be included in this quickstart
+alertPolicies:
+  - fivetran
+
+# Reference to dashboards to be included in this quickstart
+dashboards:
+  - fivetran
+dataSourceIds:
+ - fivetran
+documentation:
+  - name: Fivetran New Relic Setup Guide
+    url: https://fivetran.com/docs/logs/external-logs/new-relic/setup-guide
+    description: Setup external logging with New Relic