Skip to content

Commit 16d2462

Browse files
wpbrownwpbrown
committed
new export format support
1 parent 8c1a5d9 commit 16d2462

File tree

3 files changed

+37
-6
lines changed

3 files changed

+37
-6
lines changed

‎azuredeploy.json

+34-3
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,37 @@
394394
"apiVersion": "2018-06-01",
395395
"properties": {
396396
"activities": [
397+
{
398+
"name": "drop_old_preliminary_data",
399+
"type": "AzureDataExplorerCommand",
400+
"dependsOn": [
401+
{
402+
"activity": "copy_usage_from_archive_to_table",
403+
"dependencyConditions": [
404+
"Succeeded"
405+
]
406+
}
407+
],
408+
"policy": {
409+
"timeout": "7.00:00:00",
410+
"retry": 6,
411+
"retryIntervalInSeconds": 60,
412+
"secureOutput": false,
413+
"secureInput": false
414+
},
415+
"userProperties": [],
416+
"typeProperties": {
417+
"command": {
418+
"value": "@concat('.drop extents <| \n .show table UsagePreliminary extents \n | extend Tags = split(Tags, \"\\r\\n\")\n | where set_has_element(Tags, \"', last(split(pipeline().parameters.BlobPath, '/')),'\")')",
419+
"type": "Expression"
420+
},
421+
"commandTimeout": "00:20:00"
422+
},
423+
"linkedServiceName": {
424+
"referenceName": "link_kusto_azmeta",
425+
"type": "LinkedServiceReference"
426+
}
427+
},
397428
{
398429
"name": "copy_usage_from_archive_to_table",
399430
"type": "Copy",
@@ -584,7 +615,7 @@
584615
"type": "AzureDataExplorerSink",
585616
"ingestionMappingName": "UsagePreliminaryMapping",
586617
"additionalProperties": {
587-
"value": "@concat('{\"tags\":\"[\\\"drop-by:', pipeline().RunId, '\\\"]\"}')",
618+
"value": "@concat('{\"tags\":\"[\\\"', last(split(pipeline().parameters.BlobPath, '/')), '\\\", \\\"drop-by:', pipeline().RunId, '\\\"]\"}')",
588619
"type": "Expression"
589620
}
590621
},
@@ -630,7 +661,7 @@
630661
"userProperties": [],
631662
"typeProperties": {
632663
"command": {
633-
"value": "@concat('.drop extents <| \n .show table UsagePreliminary extents \n | extend Tags = split(Tags, \"\\r\\n\")\n | mv-expand Tags to typeof(string)\n | where Tags != \"drop-by:', pipeline().RunId,'\"')",
664+
"value": "@concat('.drop extents <| \n .show table UsagePreliminary extents \n | extend Tags = split(Tags, \"\\r\\n\")\n | where set_has_element(Tags, \"', last(split(pipeline().parameters.BlobPath, '/')),'\") and not(set_has_element(Tags, \"drop-by:', pipeline().RunId,'\"))')",
634665
"type": "Expression"
635666
},
636667
"commandTimeout": "00:20:00"
@@ -711,7 +742,7 @@
711742
],
712743
"type": "BlobEventsTrigger",
713744
"typeProperties": {
714-
"blobPathBeginsWith": "/usage-preliminary/blobs/export/finalamortized",
745+
"blobPathBeginsWith": "/usage-preliminary/blobs/export/preliminaryamortized",
715746
"blobPathEndsWith": ".csv",
716747
"ignoreEmptyBlobs": true,
717748
"scope": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]",

‎db_init/__main__.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
logging.info('Init starting.')
1010

1111
init_script = """
12-
.create table {TABLE_BASE}Ingest (BillingAccountId: int, BillingAccountName: string, BillingPeriodStartDate: datetime, BillingPeriodEndDate: datetime, BillingProfileId: int, BillingProfileName: string, AccountOwnerId: string, AccountName: string, SubscriptionId: guid, SubscriptionName: string, Date: datetime, Product: string, PartNumber: string, MeterId: guid, ServiceFamily: string, MeterCategory: string, MeterSubCategory: string, MeterRegion: string, MeterName: string, Quantity: decimal, EffectivePrice: decimal, Cost: decimal, UnitPrice: decimal, BillingCurrency: string, ResourceLocation: string, AvailabilityZone: string, ConsumedService: string, ResourceId: string, ResourceName: string, ServiceInfo1: string, ServiceInfo2: string, AdditionalInfo: dynamic, Tags: string, InvoiceSectionId: string, InvoiceSection: string, CostCenter: int, UnitOfMeasure: string, ResourceGroup: string, ReservationId: guid, ReservationName: string, ProductOrderId: guid, ProductOrderName: string, OfferId: string, IsAzureCreditEligible: string, Term: string, PublisherName: string, PlanName: string, ChargeType: string, Frequency: string, PublisherType: string)
12+
.create table {TABLE_BASE}Ingest (['InvoiceSectionName']:string, ['AccountName']:string, ['AccountOwnerId']:string, ['SubscriptionId']:string, ['SubscriptionName']:string, ['ResourceGroup']:string, ['ResourceLocation']:string, ['Date']:datetime, ['ProductName']:string, ['MeterCategory']:string, ['MeterSubCategory']:string, ['MeterId']:guid, ['MeterName']:string, ['MeterRegion']:string, ['UnitOfMeasure']:string, ['Quantity']:decimal, ['EffectivePrice']:decimal, ['CostInBillingCurrency']:decimal, ['CostCenter']:int, ['ConsumedService']:string, ['ResourceId']:string, ['Tags']:string, ['OfferId']:string, ['AdditionalInfo']:dynamic, ['ServiceInfo1']:string, ['ServiceInfo2']:string, ['ResourceName']:string, ['ReservationId']:guid, ['ReservationName']:string, ['UnitPrice']:decimal, ['ProductOrderId']:guid, ['ProductOrderName']:string, ['Term']:int, ['PublisherType']:string, ['PublisherName']:string, ['ChargeType']:string, ['Frequency']:string, ['PricingModel']:string, ['AvailabilityZone']:string, ['BillingAccountId']:int, ['BillingAccountName']:string, ['BillingCurrencyCode']:string, ['BillingPeriodStartDate']:datetime, ['BillingPeriodEndDate']:datetime, ['BillingProfileId']:int, ['BillingProfileName']:string, ['InvoiceSectionId']:string, ['IsAzureCreditEligible']:string, ['PartNumber']:string, ['PayGPrice']:decimal, ['PlanName']:string, ['ServiceFamily']:string)
1313
14-
.create table {TABLE_BASE}Ingest ingestion csv mapping '{TABLE_BASE}Mapping' '[{"Name":"BillingAccountId","DataType":"int","Ordinal":"0","ConstValue":null},{"Name":"BillingAccountName","DataType":"string","Ordinal":"1","ConstValue":null},{"Name":"BillingPeriodStartDate","DataType":"datetime","Ordinal":"2","ConstValue":null},{"Name":"BillingPeriodEndDate","DataType":"datetime","Ordinal":"3","ConstValue":null},{"Name":"BillingProfileId","DataType":"int","Ordinal":"4","ConstValue":null},{"Name":"BillingProfileName","DataType":"string","Ordinal":"5","ConstValue":null},{"Name":"AccountOwnerId","DataType":"string","Ordinal":"6","ConstValue":null},{"Name":"AccountName","DataType":"string","Ordinal":"7","ConstValue":null},{"Name":"SubscriptionId","DataType":"guid","Ordinal":"8","ConstValue":null},{"Name":"SubscriptionName","DataType":"string","Ordinal":"9","ConstValue":null},{"Name":"Date","DataType":"datetime","Ordinal":"10","ConstValue":null},{"Name":"Product","DataType":"string","Ordinal":"11","ConstValue":null},{"Name":"PartNumber","DataType":"string","Ordinal":"12","ConstValue":null},{"Name":"MeterId","DataType":"guid","Ordinal":"13","ConstValue":null},{"Name":"ServiceFamily","DataType":"string","Ordinal":"14","ConstValue":null},{"Name":"MeterCategory","DataType":"string","Ordinal":"15","ConstValue":null},{"Name":"MeterSubCategory","DataType":"string","Ordinal":"16","ConstValue":null},{"Name":"MeterRegion","DataType":"string","Ordinal":"17","ConstValue":null},{"Name":"MeterName","DataType":"string","Ordinal":"18","ConstValue":null},{"Name":"Quantity","DataType":"decimal","Ordinal":"19","ConstValue":null},{"Name":"EffectivePrice","DataType":"decimal","Ordinal":"20","ConstValue":null},{"Name":"Cost","DataType":"decimal","Ordinal":"21","ConstValue":null},{"Name":"UnitPrice","DataType":"decimal","Ordinal":"22","ConstValue":null},{"Name":"BillingCurrency","DataType":"string","Ordinal":"23","ConstValue":null},{"Name":"ResourceLocation","DataType":"string","Ordinal":"24","ConstValue":null},{"Name":"AvailabilityZone","DataType":"string","Ordinal":"25","ConstValue":null},{"Name":"ConsumedService","DataType":"string","Ordinal":"26","ConstValue":null},{"Name":"ResourceId","DataType":"string","Ordinal":"27","ConstValue":null},{"Name":"ResourceName","DataType":"string","Ordinal":"28","ConstValue":null},{"Name":"ServiceInfo1","DataType":"string","Ordinal":"29","ConstValue":null},{"Name":"ServiceInfo2","DataType":"string","Ordinal":"30","ConstValue":null},{"Name":"AdditionalInfo","DataType":"dynamic","Ordinal":"31","ConstValue":null},{"Name":"Tags","DataType":"string","Ordinal":"32","ConstValue":null},{"Name":"InvoiceSectionId","DataType":"string","Ordinal":"33","ConstValue":null},{"Name":"InvoiceSection","DataType":"string","Ordinal":"34","ConstValue":null},{"Name":"CostCenter","DataType":"int","Ordinal":"35","ConstValue":null},{"Name":"UnitOfMeasure","DataType":"string","Ordinal":"36","ConstValue":null},{"Name":"ResourceGroup","DataType":"string","Ordinal":"37","ConstValue":null},{"Name":"ReservationId","DataType":"guid","Ordinal":"38","ConstValue":null},{"Name":"ReservationName","DataType":"string","Ordinal":"39","ConstValue":null},{"Name":"ProductOrderId","DataType":"guid","Ordinal":"40","ConstValue":null},{"Name":"ProductOrderName","DataType":"string","Ordinal":"41","ConstValue":null},{"Name":"OfferId","DataType":"string","Ordinal":"42","ConstValue":null},{"Name":"IsAzureCreditEligible","DataType":"string","Ordinal":"43","ConstValue":null},{"Name":"Term","DataType":"string","Ordinal":"44","ConstValue":null},{"Name":"PublisherName","DataType":"string","Ordinal":"45","ConstValue":null},{"Name":"PlanName","DataType":"string","Ordinal":"46","ConstValue":null},{"Name":"ChargeType","DataType":"string","Ordinal":"47","ConstValue":null},{"Name":"Frequency","DataType":"string","Ordinal":"48","ConstValue":null},{"Name":"PublisherType","DataType":"string","Ordinal":"49","ConstValue":null}]'
14+
.create table {TABLE_BASE}Ingest ingestion csv mapping '{TABLE_BASE}Mapping' '[{"Name":"InvoiceSectionName","Ordinal":0},{"Name":"AccountName","Ordinal":1},{"Name":"AccountOwnerId","Ordinal":2},{"Name":"SubscriptionId","Ordinal":3},{"Name":"SubscriptionName","Ordinal":4},{"Name":"ResourceGroup","Ordinal":5},{"Name":"ResourceLocation","Ordinal":6},{"Name":"Date","Ordinal":7},{"Name":"ProductName","Ordinal":8},{"Name":"MeterCategory","Ordinal":9},{"Name":"MeterSubCategory","Ordinal":10},{"Name":"MeterId","Ordinal":11},{"Name":"MeterName","Ordinal":12},{"Name":"MeterRegion","Ordinal":13},{"Name":"UnitOfMeasure","Ordinal":14},{"Name":"Quantity","Ordinal":15},{"Name":"EffectivePrice","Ordinal":16},{"Name":"CostInBillingCurrency","Ordinal":17},{"Name":"CostCenter","Ordinal":18},{"Name":"ConsumedService","Ordinal":19},{"Name":"ResourceId","Ordinal":20},{"Name":"Tags","Ordinal":21},{"Name":"OfferId","Ordinal":22},{"Name":"AdditionalInfo","Ordinal":23},{"Name":"ServiceInfo1","Ordinal":24},{"Name":"ServiceInfo2","Ordinal":25},{"Name":"ResourceName","Ordinal":26},{"Name":"ReservationId","Ordinal":27},{"Name":"ReservationName","Ordinal":28},{"Name":"UnitPrice","Ordinal":29},{"Name":"ProductOrderId","Ordinal":30},{"Name":"ProductOrderName","Ordinal":31},{"Name":"Term","Ordinal":32},{"Name":"PublisherType","Ordinal":33},{"Name":"PublisherName","Ordinal":34},{"Name":"ChargeType","Ordinal":35},{"Name":"Frequency","Ordinal":36},{"Name":"PricingModel","Ordinal":37},{"Name":"AvailabilityZone","Ordinal":38},{"Name":"BillingAccountId","Ordinal":39},{"Name":"BillingAccountName","Ordinal":40},{"Name":"BillingCurrencyCode","Ordinal":41},{"Name":"BillingPeriodStartDate","Ordinal":42},{"Name":"BillingPeriodEndDate","Ordinal":43},{"Name":"BillingProfileId","Ordinal":44},{"Name":"BillingProfileName","Ordinal":45},{"Name":"InvoiceSectionId","Ordinal":46},{"Name":"IsAzureCreditEligible","Ordinal":47},{"Name":"PartNumber","Ordinal":48},{"Name":"PayGPrice","Ordinal":49},{"Name":"PlanName","Ordinal":50},{"Name":"ServiceFamily","Ordinal":51}]'
1515
1616
.create function
1717
IngestTo{TABLE_BASE}()

‎db_init/build_pyz.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ WORK_DIR="build/working"
55
rm -rf build
66
mkdir -p $WORK_DIR
77
cp $DIR/__main__.py $WORK_DIR
8-
pip install -r $DIR/requirements.txt --target $WORK_DIR
8+
pip3 install -r $DIR/requirements.txt --target $WORK_DIR
99
find $WORK_DIR \( -name '*.dist-info' -o -name '__pycache__' \) -prune -exec rm -r '{}' \;
1010
touch $WORK_DIR/azure/__init__.py # Workaround some problem with the .pth for Azure Namesapce.
1111
python3 -m zipapp --compress --output=build/db_init.pyz $WORK_DIR

0 commit comments

Comments
 (0)