[SIEM] [Detection Engine] Log time gaps as failures for now (#55515) (#55540)

* log a failure to failure history if time gap is detected. stop-gap solution until a feature is fully fleshed out to report this and future messaging / monitoring.

* write date the gap warning occurred in the last_failure_at field, along with the status_date field.

Author: dhurley14

x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -134,6 +134,27 @@ export const signalRulesAlertType = ({
         logger.warn(
           `Signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" has a time gap of ${gap.humanize()} (${gap.asMilliseconds()}ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.`
         );
+        // write a failure status whenever we have a time gap
+        // this is a temporary solution until general activity
+        // monitoring is developed as a feature
+        const gapDate = new Date().toISOString();
+        await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
+          alertId,
+          statusDate: gapDate,
+          status: 'failed',
+          lastFailureAt: gapDate,
+          lastSuccessAt: currentStatusSavedObject.attributes.lastSuccessAt,
+          lastFailureMessage: `Signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" has a time gap of ${gap.humanize()} (${gap.asMilliseconds()}ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.`,
+          lastSuccessMessage: currentStatusSavedObject.attributes.lastSuccessMessage,
+        });
+
+        if (ruleStatusSavedObjects.saved_objects.length >= 6) {
+          // delete fifth status and prepare to insert a newer one.
+          const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
+          await toDelete.forEach(async item =>
+            services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
+          );
+        }
       }
       // set searchAfter page size to be the lesser of default page size or maxSignals.
       const searchAfterSize =