Merge pull request #1 from Azure/master

merge from Master

Author: poanchen

eng/mgmt/mgmtmetadata/logic_resource-manager.txt

@@ -0,0 +1,14 @@
+Installing AutoRest version: latest
+AutoRest installed successfully.
+Commencing code generation
+Generating CSharp code
+Executing AutoRest command
+cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/logic/resource-manager/readme.md --csharp --version=latest --reflect-api-versions --csharp-sdks-folder=C:\dev\azure-sdk-for-net\sdk
+2019-10-15 18:51:23 UTC
+Azure-rest-api-specs repository information
+GitHub fork: Azure
+Branch:      master
+Commit:      411d4cf447639289e4ee1efa4d066df2773ebccd
+AutoRest information
+Requested version: latest
+Bootstrapper version:    [email protected]

eng/mgmt/mgmtmetadata/recoveryservicessiterecovery_resource-manager.txt

@@ -3,12 +3,12 @@ AutoRest installed successfully.
 Commencing code generation
 Generating CSharp code
 Executing AutoRest command
-cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/recoveryservicessiterecovery/resource-manager/readme.md --csharp --version=latest --reflect-api-versions --csharp-sdks-folder=D:\ASROneSDK\azure-sdk-for-net\sdk
-2019-12-20 05:05:29 UTC
+cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/recoveryservicessiterecovery/resource-manager/readme.md --csharp --version=latest --reflect-api-versions --csharp-sdks-folder=D:\Repos\AsrOneSdk-2\azure-sdk-for-net\sdk
+2020-01-02 07:34:00 UTC
 Azure-rest-api-specs repository information
 GitHub fork: Azure
 Branch:      master
-Commit:      cb5fe2f993e08939a712c20f26ff9cbab7e98c82
+Commit:      4676d9187988aa6793ff704aa39c5f495b40f6e0
 AutoRest information
 Requested version: latest
 Bootstrapper version:    [email protected]

samples/linecounter/BlobProcessorHost.cs

@@ -0,0 +1,146 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure.Messaging.EventHubs;
+using Azure.Messaging.EventHubs.Processor;
+using Azure.Storage.Blobs;
+using Microsoft.Extensions.Azure;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace LineCounter
+{
+    /// <summary>
+    /// Hosted service that listens to uploads events hub and calculates line counts for the blobs in background updating blob metadata and posting a message into results event hub
+    /// </summary>
+    public class LineCounterService : IHostedService
+    {
+        private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
+        private readonly ILogger<LineCounterService> _logger;
+
+        private readonly BlobContainerClient _blobContainerClient;
+        private readonly EventHubProducerClient _resultsClient;
+
+        private readonly EventProcessorClient _processor;
+
+        public LineCounterService(
+            IAzureClientFactory<EventHubProducerClient> producerFactory,
+            BlobServiceClient blobServiceClient,
+            ILogger<LineCounterService> logger,
+            IConfiguration configuration)
+        {
+            _logger = logger;
+            _blobContainerClient = blobServiceClient.GetBlobContainerClient("uploads");
+            _resultsClient = producerFactory.CreateClient("Results");
+            _processor = new EventProcessorClient(
+                _blobContainerClient,
+                EventHubConsumerClient.DefaultConsumerGroupName,
+                configuration["Uploads:connectionString"],
+                configuration["Uploads:eventHubName"]);
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            _logger.LogInformation("Starting processor host");
+            _processor.ProcessEventAsync += ProcessEvent;
+            _processor.ProcessErrorAsync += (eventArgs) =>
+            {
+                _logger.LogError(eventArgs.Exception, "Error in " + eventArgs.PartitionId);
+                return Task.CompletedTask;
+            };
+            await _processor.StartProcessingAsync();
+        }
+
+        private async Task ProcessEvent(ProcessEventArgs eventArgs)
+        {
+            // Note there is a bug with the HasEvent property in Preview 6, so check if Data is not null as work around.
+            // https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/eventhub/Azure.Messaging.EventHubs.Processor/samples/Sample08_EventProcessingHeartbeat.cs#L85
+            if (eventArgs.Data != null)
+            {
+                var totalCount = 0;
+                var cancellationToken = _cancellationTokenSource.Token;
+
+                var blobName = Encoding.UTF8.GetString(eventArgs.Data.Body.Span);
+
+                var blobClient = _blobContainerClient.GetBlobClient(blobName);
+                var downloadInfo = await blobClient.DownloadAsync(cancellationToken);
+
+                _logger.LogInformation("Processing {blob}", blobName);
+
+                var newLineCountingStream = new NewLineCountingStream();
+                await downloadInfo.Value.Content.CopyToAsync(newLineCountingStream, cancellationToken);
+                var newLineCount = newLineCountingStream.NewLineCount;
+
+                await blobClient.SetMetadataAsync(
+                    new Dictionary<string, string>()
+                    {
+                        { "whitespacecount", newLineCount.ToString()}
+                    },
+                    cancellationToken: cancellationToken);
+
+                _logger.LogInformation("{blob} had {lines} lines", blobName, newLineCount);
+
+                totalCount += newLineCount;
+
+                using EventDataBatch eventBatch = await _resultsClient.CreateBatchAsync();
+                eventBatch.TryAdd(new EventData(BitConverter.GetBytes(totalCount)));
+                await _resultsClient.SendAsync(eventBatch, cancellationToken);
+                await eventArgs.UpdateCheckpointAsync();
+            }
+        }
+
+        public async Task StopAsync(CancellationToken cancellationToken)
+        {
+            _cancellationTokenSource.Cancel();
+            await _processor.StopProcessingAsync();
+        }
+
+        private class NewLineCountingStream : Stream
+        {
+            public int NewLineCount { get; private set; }
+
+            public override void Flush()
+            {
+            }
+
+            public override int Read(byte[] buffer, int offset, int count)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override long Seek(long offset, SeekOrigin origin)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override void SetLength(long value)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override void Write(byte[] buffer, int offset, int count)
+            {
+                foreach (var b in buffer)
+                {
+                    if (b == (byte)'\n')
+                    {
+                        NewLineCount++;
+                    }
+                }
+            }
+
+            public override bool CanRead { get; }
+            public override bool CanSeek { get; }
+            public override bool CanWrite => true;
+            public override long Length { get; }
+            public override long Position { get; set; }
+        }
+    }
+}

samples/linecounter/Controllers/HomeController.cs

@@ -0,0 +1,66 @@
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using Azure.Messaging.EventHubs;
+using Azure.Storage.Blobs;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Azure;
+using Microsoft.Extensions.Logging;
+
+namespace LineCounter.Controllers
+{
+    [Route("/")]
+    public class HomeController : Controller
+    {
+        private readonly ILogger<HomeController> _logger;
+        private readonly BlobContainerClient _blobContainerClient;
+        private readonly EventHubProducerClient _uploadsProducer;
+
+        public HomeController(ILogger<HomeController> logger, BlobServiceClient blobServiceClient, IAzureClientFactory<EventHubProducerClient> clientFactory)
+        {
+            _logger = logger;
+            _blobContainerClient = blobServiceClient.GetBlobContainerClient("uploads");
+            _uploadsProducer = clientFactory.CreateClient("Uploads");
+        }
+
+        public IActionResult Index()
+        {
+            return View();
+        }
+
+        [HttpPost("upload")]
+        public async Task<IActionResult> Upload()
+        {
+            var names = new List<string>();
+
+            foreach (var file in HttpContext.Request.Form.Files)
+            {
+                var fileName = Guid.NewGuid().ToString("n") + " " + file.FileName;
+
+                var stream = file.OpenReadStream();
+                _logger.LogInformation("Uploading {fileName}", fileName);
+                await _blobContainerClient.CreateIfNotExistsAsync();
+                await _blobContainerClient.UploadBlobAsync(fileName, stream);
+
+                using EventDataBatch eventBatch = await _uploadsProducer.CreateBatchAsync();
+                eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(fileName)));
+                await _uploadsProducer.SendAsync(eventBatch);
+
+                names.Add(fileName);
+            }
+
+            return View("Index", names.ToArray());
+        }
+
+        [HttpGet("status/{name}.html")]
+        public async Task<string> Status(string name)
+        {
+            var properties = await _blobContainerClient.GetBlobClient(name).GetPropertiesAsync();
+            properties.Value.Metadata.TryGetValue("whitespacecount", out var count);
+
+            return count ?? "-1";
+        }
+    }
+}

samples/linecounter/LineCounter.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <LangVersion>preview</LangVersion>
+    <UserSecretsId>67ca5d37-051b-4ab1-b013-c74a2d4edb76</UserSecretsId>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Azure.Messaging.EventHubs" Version="5.0.0-preview.6" />
+    <PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.0.0-preview.6" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.1.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.12.0" />
+    <PackageReference Include="Microsoft.Extensions.Azure" Version="1.0.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Connected Services\" />
+  </ItemGroup>
+</Project>

samples/linecounter/Program.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace LineCounter
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureServices(
+                    services =>
+                    {
+                        services.AddSingleton<IHostedService, LineCounterService>();
+                    })
+                .ConfigureAppConfiguration(builder => builder.AddUserSecrets(typeof(Program).Assembly))
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder.UseStartup<Startup>();
+                });
+    }
+}

samples/linecounter/Properties/launchSettings.json

@@ -0,0 +1,27 @@
+{
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:55215/",
+      "sslPort": 0
+    }
+  },
+  "profiles": {
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "LineCounter": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "http://localhost:55217/"
+    }
+  }
+}

samples/linecounter/README.md

@@ -0,0 +1,33 @@
+# README
+
+Sample that uses illustrates using blob and event hub clients along with ASP.NET Core integration, distributed tracing and hosted services.
+
+# Configuration
+
+To run the sample set the following configuration properties using manage user secrets command in VS or user secrets command line https://docs.microsoft.com/en-us/aspnet/core/security/app-secrets?view=aspnetcore-2.2&tabs=windows#set-a-secret
+
+``` json
+{
+  "Blob": {
+    "connectionString": "..."
+  },
+  "Uploads": {
+    "connectionString": "...",
+    "eventHubName": "..."
+  },
+  "Results": {
+    "connectionString": "...",
+    "eventHubName": "..."
+  }
+}
+```
+
+To light up App Insights, add the InstrumentationKey key and value to the ApplicationInsights node in appsettings.json
+
+``` json
+{
+  "ApplicationInsights": {
+    "InstrumentationKey": "..."
+  }
+}
+```