Skip to content

Commit 0fd9dd1

Browse files
gauravmalhotgauravmalhot
committed
Update Data Downloader Sample
Align with latest SDK release
1 parent 3814571 commit 0fd9dd1

File tree

4 files changed

+78
-77
lines changed

4 files changed

+78
-77
lines changed

‎Samples/HttpDataDownloaderSample/LinkedServices/HDInsightLinkedService.json‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
2-
"name": "HDInsightLinkedService",
3-
"properties":
4-
{
5-
"type": "HDInsightBYOCLinkedService",
6-
"clusterUri": "<>",
7-
"userName": "<>",
8-
"password": "<>",
9-
"linkedServiceName": "RawEventsLinkedService"
2+
"name": "HDInsightLinkedService",
3+
"properties": {
4+
"type": "HDInsight",
5+
"typeProperties": {
6+
"clusterUri": "<>",
7+
"userName": "<>",
8+
"password": "<>",
9+
"linkedServiceName": "RawEventsLinkedService"
1010
}
11+
}
1112
}

‎Samples/HttpDataDownloaderSample/LinkedServices/RawEventsLinkedService.json‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
2-
"name": "RawEventsLinkedService",
3-
"properties":
4-
{
5-
"type": "AzureStorageLinkedService",
6-
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<>;AccountKey=<>"
2+
"name": "RawEventsLinkedService",
3+
"properties": {
4+
"type": "AzureStorage",
5+
"typeProperties": {
6+
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<>;AccountKey=<>"
77
}
8-
}
8+
}
9+
}

‎Samples/HttpDataDownloaderSample/Pipelines/DataDownloaderSamplePipeline.json‎

Lines changed: 38 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,40 @@
11
{
2-
"name": "DataDownloaderSamplePipeline",
3-
"properties":
4-
{
5-
"description" : "Custom Data Downloader from an HTTP EndPoint Sample Pipeline",
6-
"activities":
7-
[
8-
{
9-
"name": "DownloadData",
10-
"description": "Download raw data from an http endpoint",
11-
"outputs": [ {"name": "RawEventsTable"} ],
12-
"linkedServiceName": "HDInsightLinkedService",
13-
"type": "DotNetActivity",
14-
"transformation":
15-
{
16-
"assemblyName": "DataDownloaderActivity.dll",
17-
"entryPoint": "DataDownloaderActivityNS.DataDownloaderActivity",
18-
"packageLinkedService": "RawEventsLinkedService",
19-
"packageFile": "<container>/package/DataDownloaderActivity.zip",
20-
"extendedProperties":
21-
{
22-
"sliceStart": "$$Text.Format('{0:yyyyMMddHHmm}', Time.AddMinutes(SliceStart, 0))",
23-
"urlFormat": "http://dumps.wikimedia.org/other/pagecounts-raw/{0}/{0}-{1}/pagecounts-{0}{1}{2}-{3}00{4}.gz",
24-
"dataStorageAccountName": "<>",
25-
"dataStorageAccountKey": "<>",
26-
"dataStorageContainer": "<>",
27-
}
28-
},
29-
"policy":
30-
{
31-
"concurrency": 1,
32-
"executionPriorityOrder": "OldestFirst",
33-
"retry": 2,
34-
"timeout": "00:30:00"
35-
},
36-
},
2+
"name": "DataDownloaderSamplePipeline",
3+
"properties": {
4+
"description": "Custom Data Downloader from an HTTP EndPoint Sample Pipeline",
5+
"activities": [
6+
{
7+
"name": "DownloadData",
8+
"description": "Download raw data from an http endpoint",
9+
"outputs": [
10+
{
11+
"name": "RawEventsTable"
12+
}
3713
],
38-
"start": "2015-03-08T00:00:00Z",
39-
"end": "9999-09-09T00:00:00Z"
40-
}
41-
}
14+
"linkedServiceName": "HDInsightLinkedService",
15+
"type": "DotNetActivity",
16+
"typeProperties": {
17+
"assemblyName": "DataDownloaderActivity.dll",
18+
"entryPoint": "DataDownloaderActivityNS.DataDownloaderActivity",
19+
"packageLinkedService": "RawEventsLinkedService",
20+
"packageFile": "<container>/package/DataDownloaderActivity.zip",
21+
"defines": {
22+
"sliceStart": "$$Text.Format('{0:yyyyMMddHHmm}', Time.AddMinutes(SliceStart, 0))",
23+
"urlFormat": "http://dumps.wikimedia.org/other/pagecounts-raw/{0}/{0}-{1}/pagecounts-{0}{1}{2}-{3}00{4}.gz",
24+
"dataStorageAccountName": "<>",
25+
"dataStorageAccountKey": "<>",
26+
"dataStorageContainer": "<>"
27+
}
28+
},
29+
"policy": {
30+
"concurrency": 1,
31+
"executionPriorityOrder": "OldestFirst",
32+
"retry": 2,
33+
"timeout": "00:30:00"
34+
}
35+
}
36+
],
37+
"start": "2015-03-08T00:00:00Z",
38+
"end": "9999-09-09T00:00:00Z"
39+
}
40+
}

‎Samples/HttpDataDownloaderSample/Tables/RawEventsTable.json‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
1-
{
2-
"name":"RawEventsTable",
3-
"properties":{
4-
"location":{
5-
"type":"AzureBlobLocation",
6-
"folderPath":"<Container>/httpDownloadDataIn/{Slice}/",
7-
"partitionedBy":[
8-
{
9-
"name":"Slice",
10-
"value":{
11-
"type":"DateTime",
12-
"date":"SliceStart",
13-
"format":"yyyyMMddHHmm"
14-
}
15-
}
16-
],
17-
"linkedServiceName":"RawEventsLinkedService"
18-
},
19-
"availability":{
20-
"frequency":"Hour",
21-
"interval":1
22-
}
23-
}
24-
}
1+
{
2+
"name": "RawEventsTable",
3+
"properties": {
4+
"type": "AzureBlob",
5+
"linkedServiceName": "RawEventsLinkedService",
6+
"typeProperties": {
7+
"folderPath": "<Container>/httpDownloadDataIn/{Slice}/",
8+
"partitionedBy": [
9+
{
10+
"name": "Slice",
11+
"value": {
12+
"type": "DateTime",
13+
"date": "SliceStart",
14+
"format": "yyyyMMddHHmm"
15+
}
16+
}
17+
]
18+
},
19+
"availability": {
20+
"frequency": "Hour",
21+
"interval": 1
22+
}
23+
}
24+
}

0 commit comments

Comments
 (0)