diff --git a/azure/deploy-to-azure/event_hub.json b/azure/deploy-to-azure/event_hub.json index 552694d3e..84ab06753 100644 --- a/azure/deploy-to-azure/event_hub.json +++ b/azure/deploy-to-azure/event_hub.json @@ -32,7 +32,7 @@ }, "resources": [ { - "apiVersion": "2018-01-01-preview", + "apiVersion": "2024-01-01", "name": "[parameters('eventHubNamespace')]", "type": "Microsoft.EventHub/namespaces", "location": "[parameters('location')]", @@ -42,10 +42,12 @@ "capacity": 1 }, "tags": {}, - "properties": {}, + "properties": { + "minimumTlsVersion": "1.2" + }, "resources": [ { - "apiVersion": "2017-04-01", + "apiVersion": "2024-01-01", "name": "[parameters('eventHubName')]", "type": "eventhubs", "dependsOn": [ diff --git a/azure/deploy-to-azure/function_template.json b/azure/deploy-to-azure/function_template.json index 040acf712..70bc98420 100644 --- a/azure/deploy-to-azure/function_template.json +++ b/azure/deploy-to-azure/function_template.json @@ -65,16 +65,19 @@ "resources": [ { "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01", + "apiVersion": "2023-05-01", "name": "[variables('storageAccountName')]", "location": "[parameters('location')]", "kind": "StorageV2", "sku": { "name": "Standard_LRS" + }, + "properties": { + "minimumTlsVersion": "TLS1_2" } }, { - "apiVersion": "2018-11-01", + "apiVersion": "2024-04-01", "type": "Microsoft.Web/sites", "name": "[parameters('functionAppName')]", "location": "[parameters('location')]", @@ -135,7 +138,7 @@ { "name": "[concat(parameters('functionAppName'), '/', parameters('functionName'))]", "type": "Microsoft.Web/sites/functions", - "apiVersion": "2020-06-01", + "apiVersion": "2024-04-01", "properties": { "config": { "bindings": [ @@ -153,7 +156,7 @@ "disabled": false }, "files": { - "index.js": "// Unless explicitly stated otherwise all files in this repository are licensed\r\n// under the Apache License Version 2.0.\r\n// This product includes software developed at Datadog (https://www.datadoghq.com/).\r\n// Copyright 2025 Datadog, Inc.\r\n\r\nconst VERSION = '1.2.1';\r\n\r\nconst STRING = 'string'; // example: 'some message'\r\nconst STRING_ARRAY = 'string-array'; // example: ['one message', 'two message', ...]\r\nconst JSON_OBJECT = 'json-object'; // example: {\"key\": \"value\"}\r\nconst JSON_ARRAY = 'json-array'; // example: [{\"key\": \"value\"}, {\"key\": \"value\"}, ...] or [{\"records\": [{}, {}, ...]}, {\"records\": [{}, {}, ...]}, ...]\r\nconst BUFFER_ARRAY = 'buffer-array'; // example: [, ]\r\nconst JSON_STRING = 'json-string'; // example: '{\"key\": \"value\"}'\r\nconst JSON_STRING_ARRAY = 'json-string-array'; // example: ['{\"records\": [{}, {}]}'] or ['{\"key\": \"value\"}']\r\nconst INVALID = 'invalid';\r\nconst JSON_TYPE = 'json';\r\nconst STRING_TYPE = 'string';\r\n\r\nconst DD_API_KEY = process.env.DD_API_KEY || '';\r\nconst DD_SITE = process.env.DD_SITE || 'datadoghq.com';\r\nconst DD_HTTP_URL = process.env.DD_URL || 'http-intake.logs.' + DD_SITE;\r\nconst DD_HTTP_PORT = process.env.DD_PORT || 443;\r\nconst DD_REQUEST_TIMEOUT_MS = 10000;\r\nconst DD_TAGS = process.env.DD_TAGS || ''; // Replace '' by your comma-separated list of tags\r\nconst DD_SERVICE = process.env.DD_SERVICE || 'azure';\r\nconst DD_SOURCE = process.env.DD_SOURCE || 'azure';\r\nconst DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure';\r\nconst DD_PARSE_DEFENDER_LOGS = process.env.DD_PARSE_DEFENDER_LOGS; // Boolean whether to enable special parsing of Defender for Cloud logs. Set to 'false' to disable\r\n\r\nconst MAX_RETRIES = 4; // max number of times to retry a single http request\r\nconst RETRY_INTERVAL = 250; // amount of time (milliseconds) to wait before retrying request, doubles after every retry\r\n\r\n// constants relating to Defender for Cloud logs\r\nconst MSFT_DEFENDER_FOR_CLOUD = 'Microsoft Defender for Cloud';\r\nconst AZURE_SECURITY_CENTER = 'Azure Security Center';\r\nconst SECURITY_ASSESSMENTS = 'Microsoft.Security/assessments';\r\nconst SECURITY_SUB_ASSESSMENTS = 'Microsoft.Security/assessments/subAssessments';\r\nconst SECURITY_COMPLIANCE_ASSESSMENTS = 'Microsoft.Security/regulatoryComplianceStandards/regulatoryComplianceControls/regulatoryComplianceAssessments';\r\nconst SECURITY_SCORES = 'Microsoft.Security/secureScores';\r\nconst SECURITY_SCORE_CONTROLS = 'Microsoft.Security/secureScores/secureScoreControls';\r\nconst DEFENDER_FOR_CLOUD_PRODUCTS = [MSFT_DEFENDER_FOR_CLOUD, AZURE_SECURITY_CENTER];\r\nconst DEFENDER_FOR_CLOUD_RESOURCE_TYPES = [SECURITY_ASSESSMENTS, SECURITY_SUB_ASSESSMENTS, SECURITY_COMPLIANCE_ASSESSMENTS, SECURITY_SCORES, SECURITY_SCORE_CONTROLS];\r\n\r\n/*\r\nTo scrub PII from your logs, uncomment the applicable configs below. If you'd like to scrub more than just\r\nemails and IP addresses, add your own config to this map in the format\r\nNAME: {pattern: , replacement: }\r\n*/\r\nconst SCRUBBER_RULE_CONFIGS = {\r\n // REDACT_IP: {\r\n // pattern: /[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/,\r\n // replacement: 'xxx.xxx.xxx.xxx'\r\n // },\r\n // REDACT_EMAIL: {\r\n // pattern: /[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+/,\r\n // replacement: 'xxxxx@xxxxx.com'\r\n // }\r\n};\r\n\r\n/*\r\nTo split array-type fields in your logs into individual logs, you can add sections to the map below. An example of\r\na potential use case with azure.datafactory is there to show the format:\r\n{\r\n source_type:\r\n paths: [list of [list of fields in the log payload to iterate through to find the one to split]],\r\n keep_original_log: bool, if you'd like to preserve the original log in addition to the split ones or not,\r\n preserve_fields: bool, whether or not to keep the original log fields in the new split logs\r\n}\r\nYou can also set the DD_LOG_SPLITTING_CONFIG env var with a JSON string in this format.\r\n*/\r\nconst DD_LOG_SPLITTING_CONFIG = {\r\n // 'azure.datafactory': {\r\n // paths: [['properties', 'Output', 'value']],\r\n // keep_original_log: true,\r\n // preserve_fields: true\r\n // }\r\n};\r\n\r\nfunction getLogSplittingConfig() {\r\n try {\r\n return JSON.parse(process.env.DD_LOG_SPLITTING_CONFIG);\r\n } catch {\r\n return DD_LOG_SPLITTING_CONFIG;\r\n }\r\n}\r\n\r\nfunction shouldParseDefenderForCloudLogs() {\r\n // Default to true if the env variable is not set, is null, etc\r\n if (typeof DD_PARSE_DEFENDER_LOGS !== 'string') {\r\n return true;\r\n }\r\n const parse_defender_logs = DD_PARSE_DEFENDER_LOGS.toLowerCase();\r\n return !(parse_defender_logs === 'false' || parse_defender_logs === 'f');\r\n}\r\n\r\nfunction sleep(ms) {\r\n return new Promise(resolve => setTimeout(resolve, ms));\r\n}\r\n\r\nclass ScrubberRule {\r\n /**\r\n * @param {string} name\r\n * @param {string} pattern\r\n * @param {string} replacement\r\n */\r\n constructor(name, pattern, replacement) {\r\n this.name = name;\r\n this.replacement = replacement;\r\n this.regexp = RegExp(pattern, 'g');\r\n }\r\n}\r\n\r\nclass Batcher {\r\n /**\r\n * @param {number} maxItemSizeBytes\r\n * @param {number} maxBatchSizeBytes\r\n * @param {number} maxItemsCount\r\n */\r\n constructor(maxItemSizeBytes, maxBatchSizeBytes, maxItemsCount) {\r\n this.maxItemSizeBytes = maxItemSizeBytes;\r\n this.maxBatchSizeBytes = maxBatchSizeBytes;\r\n this.maxItemsCount = maxItemsCount;\r\n }\r\n\r\n batch(items) {\r\n let batches = [];\r\n let batch = [];\r\n let sizeBytes = 0;\r\n let sizeCount = 0;\r\n for (const item of items) {\r\n let itemSizeBytes = this.getSizeInBytes(item);\r\n if (\r\n sizeCount > 0 &&\r\n (sizeCount >= this.maxItemsCount ||\r\n sizeBytes + itemSizeBytes > this.maxBatchSizeBytes)\r\n ) {\r\n batches.push(batch);\r\n batch = [];\r\n sizeBytes = 0;\r\n sizeCount = 0;\r\n }\r\n // all items exceeding maxItemSizeBytes are dropped here\r\n if (itemSizeBytes <= this.maxItemSizeBytes) {\r\n batch.push(item);\r\n sizeBytes += itemSizeBytes;\r\n sizeCount += 1;\r\n }\r\n }\r\n\r\n if (sizeCount > 0) {\r\n batches.push(batch);\r\n }\r\n return batches;\r\n }\r\n\r\n getSizeInBytes(string) {\r\n if (typeof string !== 'string') {\r\n string = JSON.stringify(string);\r\n }\r\n return Buffer.byteLength(string, 'utf8');\r\n }\r\n}\r\n\r\nclass HTTPClient {\r\n /**\r\n * @param {InvocationContext} context\r\n */\r\n constructor(context) {\r\n this.context = context;\r\n this.url = `https://${DD_HTTP_URL}:${DD_HTTP_PORT}/api/v2/logs`;\r\n this.scrubber = new Scrubber(this.context, SCRUBBER_RULE_CONFIGS);\r\n this.batcher = new Batcher(256 * 1000, 4 * 1000 * 1000, 400);\r\n }\r\n\r\n async sendAll(records) {\r\n let batches = this.batcher.batch(records);\r\n return await Promise.all(\r\n batches.map(async batch => {\r\n try {\r\n return await this.send(batch);\r\n } catch (e) {\r\n this.context.log.error(e);\r\n }\r\n })\r\n );\r\n }\r\n\r\n isStatusCodeValid(statusCode) {\r\n return statusCode >= 200 && statusCode <= 299;\r\n }\r\n\r\n shouldStatusCodeRetry(statusCode) {\r\n // don't retry 4xx responses\r\n return (\r\n !this.isStatusCodeValid(statusCode) &&\r\n (statusCode < 400 || statusCode > 499)\r\n );\r\n }\r\n\r\n async send(record, retries = MAX_RETRIES, retryInterval = RETRY_INTERVAL) {\r\n const retryRequest = async errMsg => {\r\n if (retries === 0) {\r\n throw new Error(errMsg);\r\n }\r\n this.context.log.warn(\r\n `Unable to send request, with error: ${errMsg}. Retrying ${retries} more times`\r\n );\r\n retries--;\r\n retryInterval *= 2;\r\n await sleep(retryInterval);\r\n return await this.send(record, retries, retryInterval);\r\n };\r\n try {\r\n const resp = await fetch(this.url, {\r\n method: 'POST',\r\n headers: {\r\n 'Content-Type': 'application/json',\r\n 'DD-API-KEY': DD_API_KEY,\r\n 'DD-EVP-ORIGIN': 'azure'\r\n },\r\n signal: AbortSignal.timeout(DD_REQUEST_TIMEOUT_MS),\r\n body: this.scrubber.scrub(JSON.stringify(record))\r\n });\r\n if (this.isStatusCodeValid(resp.status)) {\r\n return true;\r\n } else if (this.shouldStatusCodeRetry(resp.status)) {\r\n return await retryRequest(`invalid status code ${resp.status}`);\r\n } else {\r\n throw new Error(`invalid status code ${resp.status}`);\r\n }\r\n } catch (e) {\r\n if (e.name === 'TimeoutError') {\r\n return await retryRequest(\r\n `request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`\r\n );\r\n } else {\r\n return await retryRequest(e.message);\r\n }\r\n }\r\n }\r\n}\r\n\r\nclass Scrubber {\r\n /**\r\n * @param {InvocationContext} context\r\n * @param {Record} configs\r\n */\r\n constructor(context, configs) {\r\n let rules = [];\r\n for (const [name, settings] of Object.entries(configs)) {\r\n try {\r\n rules.push(\r\n new ScrubberRule(\r\n name,\r\n settings['pattern'],\r\n settings['replacement']\r\n )\r\n );\r\n } catch {\r\n context.log.error(\r\n `Regexp for rule ${name} pattern ${settings['pattern']} is malformed, skipping. Please update the pattern for this rule to be applied.`\r\n );\r\n }\r\n }\r\n this.rules = rules;\r\n }\r\n\r\n scrub(record) {\r\n if (!this.rules) {\r\n return record;\r\n }\r\n this.rules.forEach(rule => {\r\n record = record.replace(rule.regexp, rule.replacement);\r\n });\r\n return record;\r\n }\r\n}\r\n\r\nclass EventhubLogHandler {\r\n /**\r\n * @param {InvocationContext} context\r\n */\r\n constructor(context) {\r\n this.context = context;\r\n this.logSplittingConfig = getLogSplittingConfig();\r\n this.records = [];\r\n }\r\n\r\n findSplitRecords(record, fields) {\r\n let tempRecord = record;\r\n for (const fieldName in fields) {\r\n // loop through the fields to find the one we want to split\r\n if (\r\n tempRecord[fieldName] === undefined ||\r\n tempRecord[fieldName] === null\r\n ) {\r\n // if the field is null or undefined, return\r\n return null;\r\n } else {\r\n // there is some value for the field\r\n try {\r\n // if for some reason we can't index into it, return null\r\n tempRecord = tempRecord[fieldName];\r\n } catch {\r\n return null;\r\n }\r\n }\r\n }\r\n return tempRecord;\r\n }\r\n\r\n formatLog(messageType, record) {\r\n if (messageType == JSON_TYPE) {\r\n let originalRecord = this.addTagsToJsonLog(record);\r\n // normalize the host field. Azure EventHub sends it as \"Host\".\r\n if (originalRecord.Host) {\r\n originalRecord.host = originalRecord.Host;\r\n }\r\n let source = originalRecord['ddsource'];\r\n let config = this.logSplittingConfig[source];\r\n if (config !== undefined) {\r\n let splitFieldFound = false;\r\n\r\n for (const fields of config.paths) {\r\n let recordsToSplit = this.findSplitRecords(record, fields);\r\n if (\r\n recordsToSplit === null ||\r\n !(recordsToSplit instanceof Array)\r\n ) {\r\n // if we were unable find the field or if the field isn't an array, skip it\r\n continue;\r\n }\r\n splitFieldFound = true;\r\n\r\n for (let splitRecord of recordsToSplit) {\r\n if (typeof splitRecord === 'string') {\r\n try {\r\n splitRecord = JSON.parse(splitRecord);\r\n } catch {}\r\n }\r\n let formattedSplitRecord = {};\r\n let temp = formattedSplitRecord;\r\n // re-create the same nested attributes with only the split log\r\n for (let k = 0; k < fields.length; k++) {\r\n if (k === fields.length - 1) {\r\n // if it is the last field, add the split record\r\n temp[fields[k]] = splitRecord;\r\n } else {\r\n temp[fields[k]] = {};\r\n temp = temp[fields[k]];\r\n }\r\n }\r\n formattedSplitRecord = {\r\n parsed_arrays: formattedSplitRecord\r\n };\r\n let newRecord;\r\n if (config.preserve_fields) {\r\n newRecord = { ...originalRecord };\r\n } else {\r\n newRecord = {\r\n ddsource: source,\r\n ddsourcecategory:\r\n originalRecord['ddsourcecategory'],\r\n service: originalRecord['service'],\r\n ddtags: originalRecord['ddtags']\r\n };\r\n if (originalRecord['time'] !== undefined) {\r\n newRecord['time'] = originalRecord['time'];\r\n }\r\n }\r\n Object.assign(newRecord, formattedSplitRecord);\r\n this.records.push(newRecord);\r\n }\r\n }\r\n if (config.keep_original_log || splitFieldFound !== true) {\r\n // keep the original log if it is set in the config\r\n // if it is false in the config, we should still write the log when we don't split\r\n this.records.push(originalRecord);\r\n }\r\n } else {\r\n this.records.push(originalRecord);\r\n }\r\n } else {\r\n record = this.addTagsToStringLog(record);\r\n this.records.push(record);\r\n }\r\n }\r\n\r\n handleLogs(logs) {\r\n let logsType = this.getLogFormat(logs);\r\n switch (logsType) {\r\n case STRING:\r\n this.formatLog(STRING_TYPE, logs);\r\n break;\r\n case JSON_STRING:\r\n logs = JSON.parse(logs);\r\n this.formatLog(JSON_TYPE, logs);\r\n break;\r\n case JSON_OBJECT:\r\n this.formatLog(JSON_TYPE, logs);\r\n break;\r\n case STRING_ARRAY:\r\n logs.forEach(log => this.formatLog(STRING_TYPE, log));\r\n break;\r\n case JSON_ARRAY:\r\n this.handleJSONArrayLogs(logs, JSON_ARRAY);\r\n break;\r\n case BUFFER_ARRAY:\r\n this.handleJSONArrayLogs(logs, BUFFER_ARRAY);\r\n break;\r\n case JSON_STRING_ARRAY:\r\n this.handleJSONArrayLogs(logs, JSON_STRING_ARRAY);\r\n break;\r\n case INVALID:\r\n this.context.log.error('Log format is invalid: ', logs);\r\n break;\r\n default:\r\n this.context.log.error('Log format is invalid: ', logs);\r\n break;\r\n }\r\n return this.records;\r\n }\r\n\r\n handleJSONArrayLogs(logs, logsType) {\r\n for (let message of logs) {\r\n if (logsType == JSON_STRING_ARRAY) {\r\n try {\r\n message = JSON.parse(message);\r\n } catch {\r\n this.context.log.warn(\r\n 'log is malformed json, sending as string'\r\n );\r\n this.formatLog(STRING_TYPE, message);\r\n continue;\r\n }\r\n }\r\n // If the message is a buffer object, the data type has been set to binary.\r\n if (logsType == BUFFER_ARRAY) {\r\n try {\r\n message = JSON.parse(message.toString());\r\n } catch {\r\n this.context.log.warn(\r\n 'log is malformed json, sending as string'\r\n );\r\n this.formatLog(STRING_TYPE, message.toString());\r\n continue;\r\n }\r\n }\r\n if (message.records != undefined) {\r\n message.records.forEach(message =>\r\n this.formatLog(JSON_TYPE, message)\r\n );\r\n } else {\r\n this.formatLog(JSON_TYPE, message);\r\n }\r\n }\r\n }\r\n\r\n getLogFormat(logs) {\r\n if (typeof logs === 'string') {\r\n if (this.isJsonString(logs)) {\r\n return JSON_STRING;\r\n }\r\n return STRING;\r\n }\r\n if (!Array.isArray(logs) && typeof logs === 'object' && logs !== null) {\r\n return JSON_OBJECT;\r\n }\r\n if (!Array.isArray(logs)) {\r\n return INVALID;\r\n }\r\n if (Buffer.isBuffer(logs[0])) {\r\n return BUFFER_ARRAY;\r\n }\r\n if (typeof logs[0] === 'object') {\r\n return JSON_ARRAY;\r\n }\r\n if (typeof logs[0] === 'string') {\r\n if (this.isJsonString(logs[0])) {\r\n return JSON_STRING_ARRAY;\r\n } else {\r\n return STRING_ARRAY;\r\n }\r\n }\r\n return INVALID;\r\n }\r\n\r\n isJsonString(record) {\r\n try {\r\n JSON.parse(record);\r\n return true;\r\n } catch {\r\n return false;\r\n }\r\n }\r\n\r\n createDDTags(tags) {\r\n const forwarderNameTag =\r\n 'forwardername:' + this.context.executionContext.functionName;\r\n const fowarderVersionTag = 'forwarderversion:' + VERSION;\r\n let ddTags = tags.concat([\r\n DD_TAGS,\r\n forwarderNameTag,\r\n fowarderVersionTag\r\n ]);\r\n return ddTags.filter(Boolean).join(',');\r\n }\r\n\r\n addTagsToJsonLog(record) {\r\n let [metadata, newRecord] = this.extractMetadataFromLog(record);\r\n newRecord['ddsource'] = metadata.source || DD_SOURCE;\r\n newRecord['ddsourcecategory'] = DD_SOURCE_CATEGORY;\r\n newRecord['service'] = metadata.service || DD_SERVICE;\r\n newRecord['ddtags'] = this.createDDTags(metadata.tags);\r\n return newRecord;\r\n }\r\n\r\n addTagsToStringLog(stringLog) {\r\n let jsonLog = { message: stringLog };\r\n return this.addTagsToJsonLog(jsonLog);\r\n }\r\n\r\n createResourceIdArray(resourceId) {\r\n // Convert a valid resource ID to an array, handling beginning/ending slashes\r\n let resourceIdArray = resourceId.toLowerCase().split('/');\r\n if (resourceIdArray[0] === '') {\r\n resourceIdArray = resourceIdArray.slice(1);\r\n }\r\n if (resourceIdArray[resourceIdArray.length - 1] === '') {\r\n resourceIdArray.pop();\r\n }\r\n return resourceIdArray;\r\n }\r\n\r\n isSource(resourceIdPart) {\r\n // Determine if a section of a resource ID counts as a \"source,\" in our case it means it starts with 'microsoft.'\r\n return resourceIdPart.startsWith('microsoft.');\r\n }\r\n\r\n formatSourceType(sourceType) {\r\n return sourceType.replace('microsoft.', 'azure.');\r\n }\r\n\r\n getResourceId(record) {\r\n // Most logs have resourceId, but some logs have ResourceId instead\r\n let id = record.resourceId || record.ResourceId;\r\n if (typeof id !== 'string') {\r\n return null;\r\n }\r\n return id;\r\n }\r\n\r\n extractMetadataFromLog(record) {\r\n if (shouldParseDefenderForCloudLogs() && this.isDefenderForCloudLog(record)) {\r\n return this.extractMetadataFromDefenderLog(record);\r\n }\r\n return [this.extractMetadataFromStandardLog(record), record];\r\n }\r\n\r\n extractMetadataFromStandardLog(record) {\r\n let metadata = { tags: [], source: '', service: '' };\r\n let resourceId = this.getResourceId(record);\r\n if (resourceId === null || resourceId === '') {\r\n return metadata;\r\n }\r\n resourceId = this.createResourceIdArray(resourceId);\r\n\r\n if (resourceId[0] === 'subscriptions') {\r\n if (resourceId.length > 1) {\r\n metadata.tags.push('subscription_id:' + resourceId[1]);\r\n if (resourceId.length == 2) {\r\n metadata.source = 'azure.subscription';\r\n return metadata;\r\n }\r\n }\r\n if (resourceId.length > 3) {\r\n if (\r\n resourceId[2] === 'providers' &&\r\n this.isSource(resourceId[3])\r\n ) {\r\n // handle provider-only resource IDs\r\n metadata.source = this.formatSourceType(resourceId[3]);\r\n } else {\r\n metadata.tags.push('resource_group:' + resourceId[3]);\r\n if (resourceId.length == 4) {\r\n metadata.source = 'azure.resourcegroup';\r\n return metadata;\r\n }\r\n }\r\n }\r\n if (resourceId.length > 5 && this.isSource(resourceId[5])) {\r\n metadata.source = this.formatSourceType(resourceId[5]);\r\n }\r\n } else if (resourceId[0] === 'tenants') {\r\n if (resourceId.length > 3 && resourceId[3]) {\r\n metadata.tags.push('tenant:' + resourceId[1]);\r\n metadata.source = this.formatSourceType(resourceId[3]).replace(\r\n 'aadiam',\r\n 'activedirectory'\r\n );\r\n }\r\n }\r\n return metadata;\r\n }\r\n\r\n getDefenderForCloudLogType(record) {\r\n return record.type || record.Type;\r\n }\r\n\r\n isDefenderForCloudLog(record) {\r\n const productName = record.ProductName;\r\n const type = this.getDefenderForCloudLogType(record);\r\n return DEFENDER_FOR_CLOUD_PRODUCTS.includes(productName) || DEFENDER_FOR_CLOUD_RESOURCE_TYPES.includes(type);\r\n }\r\n\r\n removeWhitespaceFromKeys(obj) {\r\n // remove whitespace from the keys of an object and capitalizes the letter that follows\r\n let newObj = {};\r\n for (const [key, value] of Object.entries(obj)) {\r\n // regex looks for word boundaries and captures the alpha character that follows\r\n const newKey = key\r\n .replace(/\\b\\w/g, c => c.toUpperCase())\r\n .replaceAll(' ', '');\r\n newObj[newKey] = value;\r\n }\r\n return newObj;\r\n }\r\n\r\n extractMetadataFromDefenderLog(record) {\r\n var metadata = { tags: [], source: 'microsoft-defender-for-cloud', service: '' };\r\n const productName = record.ProductName;\r\n const type = this.getDefenderForCloudLogType(record);\r\n\r\n if (DEFENDER_FOR_CLOUD_PRODUCTS.includes(productName)) {\r\n metadata.service = 'SecurityAlerts';\r\n const extendedProperties = record.ExtendedProperties || {};\r\n record.ExtendedProperties = this.removeWhitespaceFromKeys(extendedProperties);\r\n } else if ([SECURITY_ASSESSMENTS, SECURITY_COMPLIANCE_ASSESSMENTS].includes(type)) {\r\n metadata.service = 'SecurityRecommendations';\r\n } else if (type === SECURITY_SUB_ASSESSMENTS) {\r\n metadata.service = 'SecurityFindings';\r\n } else if ([SECURITY_SCORES, SECURITY_SCORE_CONTROLS].includes(type)) {\r\n metadata.service = 'SecureScore';\r\n } else {\r\n metadata.service = 'microsoft-defender-for-cloud';\r\n }\r\n return [metadata, record];\r\n }\r\n}\r\n\r\nmodule.exports = async function(context, eventHubMessages) {\r\n if (!DD_API_KEY || DD_API_KEY === '') {\r\n context.log.error(\r\n 'You must configure your API key before starting this function (see ## Parameters section)'\r\n );\r\n return;\r\n }\r\n let parsedLogs;\r\n try {\r\n let handler = new EventhubLogHandler(context);\r\n parsedLogs = handler.handleLogs(eventHubMessages);\r\n } catch (err) {\r\n context.log.error('Error raised when parsing logs: ', err);\r\n throw err;\r\n }\r\n let results = await new HTTPClient(context).sendAll(parsedLogs);\r\n\r\n if (results.every(v => v === true) !== true) {\r\n context.log.error(\r\n 'Some messages were unable to be sent. See other logs for details.'\r\n );\r\n }\r\n};\r\n\r\nmodule.exports.forTests = {\r\n EventhubLogHandler,\r\n Scrubber,\r\n ScrubberRule,\r\n Batcher,\r\n HTTPClient,\r\n constants: {\r\n STRING,\r\n STRING_ARRAY,\r\n JSON_OBJECT,\r\n JSON_ARRAY,\r\n BUFFER_ARRAY,\r\n JSON_STRING,\r\n JSON_STRING_ARRAY,\r\n INVALID\r\n }\r\n};\r\n" + "index.js": "// Unless explicitly stated otherwise all files in this repository are licensed\r\n// under the Apache License Version 2.0.\r\n// This product includes software developed at Datadog (https://www.datadoghq.com/).\r\n// Copyright 2025 Datadog, Inc.\r\n\r\nconst VERSION = '1.2.1';\r\n\r\nconst STRING = 'string'; // example: 'some message'\r\nconst STRING_ARRAY = 'string-array'; // example: ['one message', 'two message', ...]\r\nconst JSON_OBJECT = 'json-object'; // example: {\"key\": \"value\"}\r\nconst JSON_ARRAY = 'json-array'; // example: [{\"key\": \"value\"}, {\"key\": \"value\"}, ...] or [{\"records\": [{}, {}, ...]}, {\"records\": [{}, {}, ...]}, ...]\r\nconst BUFFER_ARRAY = 'buffer-array'; // example: [, ]\r\nconst JSON_STRING = 'json-string'; // example: '{\"key\": \"value\"}'\r\nconst JSON_STRING_ARRAY = 'json-string-array'; // example: ['{\"records\": [{}, {}]}'] or ['{\"key\": \"value\"}']\r\nconst INVALID = 'invalid';\r\nconst JSON_TYPE = 'json';\r\nconst STRING_TYPE = 'string';\r\n\r\nconst DD_API_KEY = process.env.DD_API_KEY || '';\r\nconst DD_SITE = process.env.DD_SITE || 'datadoghq.com';\r\nconst DD_HTTP_URL = process.env.DD_URL || 'http-intake.logs.' + DD_SITE;\r\nconst DD_HTTP_PORT = process.env.DD_PORT || 443;\r\nconst DD_REQUEST_TIMEOUT_MS = 10000;\r\nconst DD_TAGS = process.env.DD_TAGS || ''; // Replace '' by your comma-separated list of tags\r\nconst DD_SERVICE = process.env.DD_SERVICE || 'azure';\r\nconst DD_SOURCE = process.env.DD_SOURCE || 'azure';\r\nconst DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure';\r\nconst DD_PARSE_DEFENDER_LOGS = process.env.DD_PARSE_DEFENDER_LOGS; // Boolean whether to enable special parsing of Defender for Cloud logs. Set to 'false' to disable\r\n\r\nconst MAX_RETRIES = 4; // max number of times to retry a single http request\r\nconst RETRY_INTERVAL = 250; // amount of time (milliseconds) to wait before retrying request, doubles after every retry\r\n\r\n// constants relating to Defender for Cloud logs\r\nconst MSFT_DEFENDER_FOR_CLOUD = 'Microsoft Defender for Cloud';\r\nconst AZURE_SECURITY_CENTER = 'Azure Security Center';\r\nconst SECURITY_ASSESSMENTS = 'Microsoft.Security/assessments';\r\nconst SECURITY_SUB_ASSESSMENTS = 'Microsoft.Security/assessments/subAssessments';\r\nconst SECURITY_COMPLIANCE_ASSESSMENTS = 'Microsoft.Security/regulatoryComplianceStandards/regulatoryComplianceControls/regulatoryComplianceAssessments';\r\nconst SECURITY_SCORES = 'Microsoft.Security/secureScores';\r\nconst SECURITY_SCORE_CONTROLS = 'Microsoft.Security/secureScores/secureScoreControls';\r\nconst DEFENDER_FOR_CLOUD_PRODUCTS = [MSFT_DEFENDER_FOR_CLOUD, AZURE_SECURITY_CENTER];\r\nconst DEFENDER_FOR_CLOUD_RESOURCE_TYPES = [SECURITY_ASSESSMENTS, SECURITY_SUB_ASSESSMENTS, SECURITY_COMPLIANCE_ASSESSMENTS, SECURITY_SCORES, SECURITY_SCORE_CONTROLS];\r\n\r\n/*\r\nTo scrub PII from your logs, uncomment the applicable configs below. If you'd like to scrub more than just\r\nemails and IP addresses, add your own config to this map in the format\r\nNAME: {pattern: , replacement: }\r\n*/\r\nconst SCRUBBER_RULE_CONFIGS = {\r\n // REDACT_IP: {\r\n // pattern: /[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/,\r\n // replacement: 'xxx.xxx.xxx.xxx'\r\n // },\r\n // REDACT_EMAIL: {\r\n // pattern: /[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+/,\r\n // replacement: 'xxxxx@xxxxx.com'\r\n // }\r\n};\r\n\r\n/*\r\nTo split array-type fields in your logs into individual logs, you can add sections to the map below. An example of\r\na potential use case with azure.datafactory is there to show the format:\r\n{\r\n source_type:\r\n paths: [list of [list of fields in the log payload to iterate through to find the one to split]],\r\n keep_original_log: bool, if you'd like to preserve the original log in addition to the split ones or not,\r\n preserve_fields: bool, whether or not to keep the original log fields in the new split logs\r\n}\r\nYou can also set the DD_LOG_SPLITTING_CONFIG env var with a JSON string in this format.\r\n*/\r\nconst DD_LOG_SPLITTING_CONFIG = {\r\n // 'azure.datafactory': {\r\n // paths: [['properties', 'Output', 'value']],\r\n // keep_original_log: true,\r\n // preserve_fields: true\r\n // }\r\n};\r\n\r\nfunction getLogSplittingConfig() {\r\n try {\r\n return JSON.parse(process.env.DD_LOG_SPLITTING_CONFIG);\r\n } catch {\r\n return DD_LOG_SPLITTING_CONFIG;\r\n }\r\n}\r\n\r\nfunction shouldParseDefenderForCloudLogs() {\r\n // Default to true if the env variable is not set, is null, etc\r\n if (typeof DD_PARSE_DEFENDER_LOGS !== 'string') {\r\n return true;\r\n }\r\n const parse_defender_logs = DD_PARSE_DEFENDER_LOGS.toLowerCase();\r\n return !(parse_defender_logs === 'false' || parse_defender_logs === 'f');\r\n}\r\n\r\nfunction sleep(ms) {\r\n return new Promise(resolve => setTimeout(resolve, ms));\r\n}\r\n\r\nclass ScrubberRule {\r\n /**\r\n * @param {string} name\r\n * @param {string} pattern\r\n * @param {string} replacement\r\n */\r\n constructor(name, pattern, replacement) {\r\n this.name = name;\r\n this.replacement = replacement;\r\n this.regexp = RegExp(pattern, 'g');\r\n }\r\n}\r\n\r\nclass Batcher {\r\n /**\r\n * @param {number} maxItemSizeBytes\r\n * @param {number} maxBatchSizeBytes\r\n * @param {number} maxItemsCount\r\n */\r\n constructor(maxItemSizeBytes, maxBatchSizeBytes, maxItemsCount) {\r\n this.maxItemSizeBytes = maxItemSizeBytes;\r\n this.maxBatchSizeBytes = maxBatchSizeBytes;\r\n this.maxItemsCount = maxItemsCount;\r\n }\r\n\r\n batch(items) {\r\n let batches = [];\r\n let batch = [];\r\n let sizeBytes = 0;\r\n let sizeCount = 0;\r\n for (const item of items) {\r\n let itemSizeBytes = this.getSizeInBytes(item);\r\n if (\r\n sizeCount > 0 &&\r\n (sizeCount >= this.maxItemsCount ||\r\n sizeBytes + itemSizeBytes > this.maxBatchSizeBytes)\r\n ) {\r\n batches.push(batch);\r\n batch = [];\r\n sizeBytes = 0;\r\n sizeCount = 0;\r\n }\r\n // all items exceeding maxItemSizeBytes are dropped here\r\n if (itemSizeBytes <= this.maxItemSizeBytes) {\r\n batch.push(item);\r\n sizeBytes += itemSizeBytes;\r\n sizeCount += 1;\r\n }\r\n }\r\n\r\n if (sizeCount > 0) {\r\n batches.push(batch);\r\n }\r\n return batches;\r\n }\r\n\r\n getSizeInBytes(string) {\r\n if (typeof string !== 'string') {\r\n string = JSON.stringify(string);\r\n }\r\n return Buffer.byteLength(string, 'utf8');\r\n }\r\n}\r\n\r\nclass HTTPClient {\r\n /**\r\n * @param {InvocationContext} context\r\n */\r\n constructor(context) {\r\n this.context = context;\r\n this.url = `https://${DD_HTTP_URL}:${DD_HTTP_PORT}/api/v2/logs`;\r\n this.scrubber = new Scrubber(this.context, SCRUBBER_RULE_CONFIGS);\r\n this.batcher = new Batcher(256 * 1000, 4 * 1000 * 1000, 400);\r\n }\r\n\r\n async sendAll(records) {\r\n let batches = this.batcher.batch(records);\r\n return await Promise.all(\r\n batches.map(async batch => {\r\n try {\r\n return await this.send(batch);\r\n } catch (e) {\r\n this.context.log.error(e);\r\n }\r\n })\r\n );\r\n }\r\n\r\n isStatusCodeValid(statusCode) {\r\n return statusCode >= 200 && statusCode <= 299;\r\n }\r\n\r\n shouldStatusCodeRetry(statusCode) {\r\n // don't retry 4xx responses\r\n return (\r\n !this.isStatusCodeValid(statusCode) &&\r\n (statusCode < 400 || statusCode > 499)\r\n );\r\n }\r\n\r\n async send(record, retries = MAX_RETRIES, retryInterval = RETRY_INTERVAL) {\r\n const retryRequest = async errMsg => {\r\n if (retries === 0) {\r\n throw new Error(errMsg);\r\n }\r\n this.context.log.warn(\r\n `Unable to send request, with error: ${errMsg}. Retrying ${retries} more times`\r\n );\r\n retries--;\r\n retryInterval *= 2;\r\n await sleep(retryInterval);\r\n return await this.send(record, retries, retryInterval);\r\n };\r\n try {\r\n const resp = await fetch(this.url, {\r\n method: 'POST',\r\n headers: {\r\n 'Content-Type': 'application/json',\r\n 'DD-API-KEY': DD_API_KEY,\r\n 'DD-EVP-ORIGIN': 'azure'\r\n },\r\n signal: AbortSignal.timeout(DD_REQUEST_TIMEOUT_MS),\r\n body: this.scrubber.scrub(JSON.stringify(record))\r\n });\r\n if (this.isStatusCodeValid(resp.status)) {\r\n return true;\r\n } else if (this.shouldStatusCodeRetry(resp.status)) {\r\n return await retryRequest(`invalid status code ${resp.status}`);\r\n } else {\r\n throw new Error(`invalid status code ${resp.status}`);\r\n }\r\n } catch (e) {\r\n if (e.name === 'TimeoutError') {\r\n return await retryRequest(\r\n `request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`\r\n );\r\n } else {\r\n return await retryRequest(e.message);\r\n }\r\n }\r\n }\r\n}\r\n\r\nclass Scrubber {\r\n /**\r\n * @param {InvocationContext} context\r\n * @param {Record} configs\r\n */\r\n constructor(context, configs) {\r\n let rules = [];\r\n for (const [name, settings] of Object.entries(configs)) {\r\n try {\r\n rules.push(\r\n new ScrubberRule(\r\n name,\r\n settings['pattern'],\r\n settings['replacement']\r\n )\r\n );\r\n } catch {\r\n context.log.error(\r\n `Regexp for rule ${name} pattern ${settings['pattern']} is malformed, skipping. Please update the pattern for this rule to be applied.`\r\n );\r\n }\r\n }\r\n this.rules = rules;\r\n }\r\n\r\n scrub(record) {\r\n if (!this.rules) {\r\n return record;\r\n }\r\n this.rules.forEach(rule => {\r\n record = record.replace(rule.regexp, rule.replacement);\r\n });\r\n return record;\r\n }\r\n}\r\n\r\nclass EventhubLogHandler {\r\n /**\r\n * @param {InvocationContext} context\r\n */\r\n constructor(context) {\r\n this.context = context;\r\n this.logSplittingConfig = getLogSplittingConfig();\r\n this.records = [];\r\n }\r\n\r\n findSplitRecords(record, fields) {\r\n let tempRecord = record;\r\n for (const fieldName in fields) {\r\n // loop through the fields to find the one we want to split\r\n if (\r\n tempRecord[fieldName] === undefined ||\r\n tempRecord[fieldName] === null\r\n ) {\r\n // if the field is null or undefined, return\r\n return null;\r\n } else {\r\n // there is some value for the field\r\n try {\r\n // if for some reason we can't index into it, return null\r\n tempRecord = tempRecord[fieldName];\r\n } catch {\r\n return null;\r\n }\r\n }\r\n }\r\n return tempRecord;\r\n }\r\n\r\n formatLog(messageType, record) {\r\n if (messageType == JSON_TYPE) {\r\n let originalRecord = this.addTagsToJsonLog(record);\r\n // normalize the host field. Azure EventHub sends it as \"Host\".\r\n if (originalRecord.Host) {\r\n originalRecord.host = originalRecord.Host;\r\n }\r\n let source = originalRecord['ddsource'];\r\n let config = this.logSplittingConfig[source];\r\n if (config !== undefined) {\r\n let splitFieldFound = false;\r\n\r\n for (const fields of config.paths) {\r\n let recordsToSplit = this.findSplitRecords(record, fields);\r\n if (\r\n recordsToSplit === null ||\r\n !(recordsToSplit instanceof Array)\r\n ) {\r\n // if we were unable find the field or if the field isn't an array, skip it\r\n continue;\r\n }\r\n splitFieldFound = true;\r\n\r\n for (let splitRecord of recordsToSplit) {\r\n if (typeof splitRecord === 'string') {\r\n try {\r\n splitRecord = JSON.parse(splitRecord);\r\n } catch {}\r\n }\r\n let formattedSplitRecord = {};\r\n let temp = formattedSplitRecord;\r\n // re-create the same nested attributes with only the split log\r\n for (let k = 0; k < fields.length; k++) {\r\n if (k === fields.length - 1) {\r\n // if it is the last field, add the split record\r\n temp[fields[k]] = splitRecord;\r\n } else {\r\n temp[fields[k]] = {};\r\n temp = temp[fields[k]];\r\n }\r\n }\r\n formattedSplitRecord = {\r\n parsed_arrays: formattedSplitRecord\r\n };\r\n let newRecord;\r\n if (config.preserve_fields) {\r\n newRecord = { ...originalRecord };\r\n } else {\r\n newRecord = {\r\n ddsource: source,\r\n ddsourcecategory:\r\n originalRecord['ddsourcecategory'],\r\n service: originalRecord['service'],\r\n ddtags: originalRecord['ddtags']\r\n };\r\n if (originalRecord['time'] !== undefined) {\r\n newRecord['time'] = originalRecord['time'];\r\n }\r\n }\r\n Object.assign(newRecord, formattedSplitRecord);\r\n this.records.push(newRecord);\r\n }\r\n }\r\n if (config.keep_original_log || splitFieldFound !== true) {\r\n // keep the original log if it is set in the config\r\n // if it is false in the config, we should still write the log when we don't split\r\n this.records.push(originalRecord);\r\n }\r\n } else {\r\n this.records.push(originalRecord);\r\n }\r\n } else {\r\n record = this.addTagsToStringLog(record);\r\n this.records.push(record);\r\n }\r\n }\r\n\r\n handleLogs(logs) {\r\n let logsType = this.getLogFormat(logs);\r\n switch (logsType) {\r\n case STRING:\r\n this.formatLog(STRING_TYPE, logs);\r\n break;\r\n case JSON_STRING:\r\n logs = JSON.parse(logs);\r\n this.formatLog(JSON_TYPE, logs);\r\n break;\r\n case JSON_OBJECT:\r\n this.formatLog(JSON_TYPE, logs);\r\n break;\r\n case STRING_ARRAY:\r\n logs.forEach(log => this.formatLog(STRING_TYPE, log));\r\n break;\r\n case JSON_ARRAY:\r\n this.handleJSONArrayLogs(logs, JSON_ARRAY);\r\n break;\r\n case BUFFER_ARRAY:\r\n this.handleJSONArrayLogs(logs, BUFFER_ARRAY);\r\n break;\r\n case JSON_STRING_ARRAY:\r\n this.handleJSONArrayLogs(logs, JSON_STRING_ARRAY);\r\n break;\r\n case INVALID:\r\n this.context.log.error('Log format is invalid: ', logs);\r\n break;\r\n default:\r\n this.context.log.error('Log format is invalid: ', logs);\r\n break;\r\n }\r\n return this.records;\r\n }\r\n\r\n handleJSONArrayLogs(logs, logsType) {\r\n for (let message of logs) {\r\n if (logsType == JSON_STRING_ARRAY) {\r\n try {\r\n message = JSON.parse(message);\r\n } catch {\r\n this.context.log.warn(\r\n 'log is malformed json, sending as string'\r\n );\r\n this.formatLog(STRING_TYPE, message);\r\n continue;\r\n }\r\n }\r\n // If the message is a buffer object, the data type has been set to binary.\r\n if (logsType == BUFFER_ARRAY) {\r\n try {\r\n message = JSON.parse(message.toString());\r\n } catch {\r\n this.context.log.warn(\r\n 'log is malformed json, sending as string'\r\n );\r\n this.formatLog(STRING_TYPE, message.toString());\r\n continue;\r\n }\r\n }\r\n if (message.records != undefined) {\r\n message.records.forEach(message =>\r\n this.formatLog(JSON_TYPE, message)\r\n );\r\n } else {\r\n this.formatLog(JSON_TYPE, message);\r\n }\r\n }\r\n }\r\n\r\n getLogFormat(logs) {\r\n if (typeof logs === 'string') {\r\n if (this.isJsonString(logs)) {\r\n return JSON_STRING;\r\n }\r\n return STRING;\r\n }\r\n if (!Array.isArray(logs) && typeof logs === 'object' && logs !== null) {\r\n return JSON_OBJECT;\r\n }\r\n if (!Array.isArray(logs)) {\r\n return INVALID;\r\n }\r\n if (Buffer.isBuffer(logs[0])) {\r\n return BUFFER_ARRAY;\r\n }\r\n if (typeof logs[0] === 'object') {\r\n return JSON_ARRAY;\r\n }\r\n if (typeof logs[0] === 'string') {\r\n if (this.isJsonString(logs[0])) {\r\n return JSON_STRING_ARRAY;\r\n } else {\r\n return STRING_ARRAY;\r\n }\r\n }\r\n return INVALID;\r\n }\r\n\r\n isJsonString(record) {\r\n try {\r\n JSON.parse(record);\r\n return true;\r\n } catch {\r\n return false;\r\n }\r\n }\r\n\r\n createDDTags(tags) {\r\n const forwarderNameTag =\r\n 'forwardername:' + this.context.executionContext.functionName;\r\n const fowarderVersionTag = 'forwarderversion:' + VERSION;\r\n let ddTags = tags.concat([\r\n DD_TAGS,\r\n forwarderNameTag,\r\n fowarderVersionTag\r\n ]);\r\n return ddTags.filter(Boolean).join(',');\r\n }\r\n\r\n addTagsToJsonLog(record) {\r\n let [metadata, newRecord] = this.extractMetadataFromLog(record);\r\n newRecord['ddsource'] = metadata.source || DD_SOURCE;\r\n newRecord['ddsourcecategory'] = DD_SOURCE_CATEGORY;\r\n newRecord['service'] = metadata.service || DD_SERVICE;\r\n newRecord['ddtags'] = this.createDDTags(metadata.tags);\r\n return newRecord;\r\n }\r\n\r\n addTagsToStringLog(stringLog) {\r\n let jsonLog = { message: stringLog };\r\n return this.addTagsToJsonLog(jsonLog);\r\n }\r\n\r\n createResourceIdArray(resourceId) {\r\n // Convert a valid resource ID to an array, handling beginning/ending slashes\r\n let resourceIdArray = resourceId.toLowerCase().split('/');\r\n if (resourceIdArray[0] === '') {\r\n resourceIdArray = resourceIdArray.slice(1);\r\n }\r\n if (resourceIdArray[resourceIdArray.length - 1] === '') {\r\n resourceIdArray.pop();\r\n }\r\n return resourceIdArray;\r\n }\r\n\r\n isSource(resourceIdPart) {\r\n // Determine if a section of a resource ID counts as a \"source,\" in our case it means it starts with 'microsoft.'\r\n return resourceIdPart.startsWith('microsoft.');\r\n }\r\n\r\n formatSourceType(sourceType) {\r\n return sourceType.replace('microsoft.', 'azure.');\r\n }\r\n\r\n getResourceId(record) {\r\n // Most logs have resourceId, but some logs have ResourceId instead\r\n let id = record.resourceId || record.ResourceId;\r\n if (typeof id !== 'string') {\r\n return null;\r\n }\r\n return id;\r\n }\r\n\r\n extractMetadataFromLog(record) {\r\n if (shouldParseDefenderForCloudLogs() && this.isDefenderForCloudLog(record)) {\r\n return this.extractMetadataFromDefenderLog(record);\r\n }\r\n return [this.extractMetadataFromStandardLog(record), record];\r\n }\r\n\r\n extractMetadataFromStandardLog(record) {\r\n let metadata = { tags: [], source: '', service: '' };\r\n let resourceId = this.getResourceId(record);\r\n if (resourceId === null || resourceId === '') {\r\n return metadata;\r\n }\r\n resourceId = this.createResourceIdArray(resourceId);\r\n\r\n if (resourceId[0] === 'subscriptions') {\r\n if (resourceId.length > 1) {\r\n metadata.tags.push('subscription_id:' + resourceId[1]);\r\n if (resourceId.length == 2) {\r\n metadata.source = 'azure.subscription';\r\n return metadata;\r\n }\r\n }\r\n if (resourceId.length > 3) {\r\n if (\r\n resourceId[2] === 'providers' &&\r\n this.isSource(resourceId[3])\r\n ) {\r\n // handle provider-only resource IDs\r\n metadata.source = this.formatSourceType(resourceId[3]);\r\n } else {\r\n metadata.tags.push('resource_group:' + resourceId[3]);\r\n if (resourceId.length == 4) {\r\n metadata.source = 'azure.resourcegroup';\r\n return metadata;\r\n }\r\n }\r\n }\r\n if (resourceId.length > 5 && this.isSource(resourceId[5])) {\r\n metadata.source = this.formatSourceType(resourceId[5]);\r\n }\r\n } else if (resourceId[0] === 'tenants') {\r\n if (resourceId.length > 3 && resourceId[3]) {\r\n metadata.tags.push('tenant:' + resourceId[1]);\r\n metadata.source = this.formatSourceType(resourceId[3]).replace(\r\n 'aadiam',\r\n 'activedirectory'\r\n );\r\n }\r\n }\r\n return metadata;\r\n }\r\n\r\n getDefenderForCloudLogType(record) {\r\n return record.type || record.Type;\r\n }\r\n\r\n isDefenderForCloudLog(record) {\r\n const productName = record.ProductName;\r\n const type = this.getDefenderForCloudLogType(record);\r\n return DEFENDER_FOR_CLOUD_PRODUCTS.includes(productName) || DEFENDER_FOR_CLOUD_RESOURCE_TYPES.includes(type);\r\n }\r\n\r\n removeWhitespaceFromKeys(obj) {\r\n // remove whitespace from the keys of an object and capitalizes the letter that follows\r\n let newObj = {};\r\n for (const [key, value] of Object.entries(obj)) {\r\n // regex looks for word boundaries and captures the alpha character that follows\r\n const newKey = key\r\n .replace(/\\b\\w/g, c => c.toUpperCase())\r\n .replaceAll(' ', '');\r\n newObj[newKey] = value;\r\n }\r\n return newObj;\r\n }\r\n\r\n extractMetadataFromDefenderLog(record) {\r\n var metadata = { tags: [], source: 'microsoft-defender-for-cloud', service: '' };\r\n const productName = record.ProductName;\r\n const type = this.getDefenderForCloudLogType(record);\r\n\r\n if (DEFENDER_FOR_CLOUD_PRODUCTS.includes(productName)) {\r\n metadata.service = 'SecurityAlerts';\r\n const extendedProperties = record.ExtendedProperties || {};\r\n record.ExtendedProperties = this.removeWhitespaceFromKeys(extendedProperties);\r\n } else if ([SECURITY_ASSESSMENTS, SECURITY_COMPLIANCE_ASSESSMENTS].includes(type)) {\r\n metadata.service = 'SecurityRecommendations';\r\n } else if (type === SECURITY_SUB_ASSESSMENTS) {\r\n metadata.service = 'SecurityFindings';\r\n } else if ([SECURITY_SCORES, SECURITY_SCORE_CONTROLS].includes(type)) {\r\n metadata.service = 'SecureScore';\r\n } else {\r\n metadata.service = 'microsoft-defender-for-cloud';\r\n }\r\n return [metadata, record];\r\n }\r\n}\r\n\r\nmodule.exports = async function(context, eventHubMessages) {\r\n if (!DD_API_KEY || DD_API_KEY === '') {\r\n context.log.error(\r\n 'You must configure your API key before starting this function (see ## Parameters section)'\r\n );\r\n return;\r\n }\r\n let parsedLogs;\r\n try {\r\n let handler = new EventhubLogHandler(context);\r\n parsedLogs = handler.handleLogs(eventHubMessages);\r\n } catch (err) {\r\n context.log.error('Error raised when parsing logs: ', err);\r\n throw err;\r\n }\r\n let results = await new HTTPClient(context).sendAll(parsedLogs);\r\n\r\n if (results.every(v => v === true) !== true) {\r\n context.log.error(\r\n 'Some messages were unable to be sent. See other logs for details.'\r\n );\r\n }\r\n};\r\n\r\nmodule.exports.forTests = {\r\n EventhubLogHandler,\r\n Scrubber,\r\n ScrubberRule,\r\n Batcher,\r\n HTTPClient,\r\n constants: {\r\n STRING,\r\n STRING_ARRAY,\r\n JSON_OBJECT,\r\n JSON_ARRAY,\r\n BUFFER_ARRAY,\r\n JSON_STRING,\r\n JSON_STRING_ARRAY,\r\n INVALID\r\n }\r\n};\r\n" } }, "dependsOn": [ diff --git a/azure/deploy-to-azure/parent_template.json b/azure/deploy-to-azure/parent_template.json index 19e7ffc0f..34cbbe019 100644 --- a/azure/deploy-to-azure/parent_template.json +++ b/azure/deploy-to-azure/parent_template.json @@ -72,11 +72,11 @@ } }, "diagnosticSettingName": { - "type": "string", - "defaultValue": "[concat('datadog-activity-logs-diagnostic-setting-', uniqueString(newGuid()))]", - "metadata": { - "description": "The name of the diagnostic setting if sending Activity Logs" - } + "type": "string", + "defaultValue": "[concat('datadog-activity-logs-diagnostic-setting-', uniqueString(newGuid()))]", + "metadata": { + "description": "The name of the diagnostic setting if sending Activity Logs" + } } }, "variables": { @@ -185,10 +185,10 @@ "location": "[parameters('resourcesLocation')]" } ], - "outputs": { - "eventHubNamespace": { - "type": "string", - "value": "[parameters('eventHubNamespace')]" - } + "outputs": { + "eventHubNamespace": { + "type": "string", + "value": "[parameters('eventHubNamespace')]" } + } } \ No newline at end of file diff --git a/azure/eventhub_log_forwarder/event_hub.json b/azure/eventhub_log_forwarder/event_hub.json index ed462f7bb..40f5c3433 100644 --- a/azure/eventhub_log_forwarder/event_hub.json +++ b/azure/eventhub_log_forwarder/event_hub.json @@ -25,7 +25,7 @@ }, "resources": [ { - "apiVersion": "2018-01-01-preview", + "apiVersion": "2024-01-01", "name": "[parameters('eventHubNamespace')]", "type": "Microsoft.EventHub/namespaces", "location": "[parameters('location')]", @@ -35,10 +35,12 @@ "capacity": 1 }, "tags": {}, - "properties": {}, + "properties": { + "minimumTlsVersion": "1.2" + }, "resources": [ { - "apiVersion": "2017-04-01", + "apiVersion": "2024-01-01", "name": "[parameters('eventHubName')]", "type": "eventhubs", "dependsOn": [ diff --git a/azure/eventhub_log_forwarder/function_template.json b/azure/eventhub_log_forwarder/function_template.json index 149b6dbe1..5b279d0f2 100644 --- a/azure/eventhub_log_forwarder/function_template.json +++ b/azure/eventhub_log_forwarder/function_template.json @@ -78,16 +78,19 @@ "resources": [ { "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2019-06-01", + "apiVersion": "2023-05-01", "name": "[variables('storageAccountName')]", "location": "[parameters('location')]", "kind": "StorageV2", "sku": { "name": "Standard_LRS" + }, + "properties": { + "minimumTlsVersion": "TLS1_2" } }, { - "apiVersion": "2018-11-01", + "apiVersion": "2024-04-01", "type": "Microsoft.Web/sites", "name": "[parameters('functionAppName')]", "location": "[parameters('location')]", @@ -152,7 +155,7 @@ { "name": "[concat(parameters('functionAppName'), '/', parameters('functionName'))]", "type": "Microsoft.Web/sites/functions", - "apiVersion": "2020-06-01", + "apiVersion": "2024-04-01", "properties": { "config": { "bindings": [ diff --git a/azure/eventhub_log_forwarder/parent_template.json b/azure/eventhub_log_forwarder/parent_template.json index 4c3ac53b6..055148737 100644 --- a/azure/eventhub_log_forwarder/parent_template.json +++ b/azure/eventhub_log_forwarder/parent_template.json @@ -79,7 +79,7 @@ { "name": "eventHubTemplate", "type": "Microsoft.Resources/deployments", - "apiVersion": "2018-05-01", + "apiVersion": "2024-07-01", "properties": { "mode": "Incremental", "templateLink": { @@ -102,7 +102,7 @@ { "name": "functionAppTemplate", "type": "Microsoft.Resources/deployments", - "apiVersion": "2018-05-01", + "apiVersion": "2024-07-01", "properties": { "mode": "Incremental", "templateLink": { @@ -147,10 +147,10 @@ ] } ], - "outputs": { - "eventHubNamespace": { - "type": "string", - "value": "[parameters('eventHubNamespace')]" - } + "outputs": { + "eventHubNamespace": { + "type": "string", + "value": "[parameters('eventHubNamespace')]" } + } }