diff --git a/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj b/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj index 515ffe220eb..9586ddad4bf 100644 --- a/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj +++ b/src/dotnet/APIView/APIViewWeb/APIViewWeb.csproj @@ -27,6 +27,12 @@ + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/src/dotnet/APIView/APIViewWeb/Controllers/ReviewController.cs b/src/dotnet/APIView/APIViewWeb/Controllers/ReviewController.cs new file mode 100644 index 00000000000..4d0ebc8019b --- /dev/null +++ b/src/dotnet/APIView/APIViewWeb/Controllers/ReviewController.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using APIViewWeb.Filters; +using APIViewWeb.Repositories; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace APIViewWeb.Controllers +{ + public class ReviewController : Controller + { + private readonly ReviewManager _reviewManager; + private readonly ILogger _logger; + + public ReviewController(ReviewManager reviewManager, ILogger logger) + { + _reviewManager = reviewManager; + _logger = logger; + } + + [HttpGet] + public async Task UpdateApiReview(string repoName, string artifactPath, string buildId, string project = "internal") + { + await _reviewManager.UpdateReviewCodeFiles(repoName, buildId, artifactPath, project); + return Ok(); + } + } +} diff --git a/src/dotnet/APIView/APIViewWeb/Languages/LanguageProcessor.cs b/src/dotnet/APIView/APIViewWeb/Languages/LanguageProcessor.cs index 3dc0ff9c885..24606bae878 100644 --- a/src/dotnet/APIView/APIViewWeb/Languages/LanguageProcessor.cs +++ b/src/dotnet/APIView/APIViewWeb/Languages/LanguageProcessor.cs @@ -1,9 +1,12 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text.Json; using System.Threading.Tasks; using ApiView; - +using APIViewWeb.Models; + namespace APIViewWeb { public abstract class LanguageProcessor: LanguageService diff --git a/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs b/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs index 910761cfb21..c2b6b341676 100644 --- a/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs +++ b/src/dotnet/APIView/APIViewWeb/Languages/LanguageService.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using ApiView; +using APIView; +using APIViewWeb.Models; namespace APIViewWeb { @@ -12,5 +15,16 @@ public abstract class LanguageService public virtual bool IsSupportedFile(string name) => name.EndsWith(Extension, StringComparison.OrdinalIgnoreCase); public abstract bool CanUpdate(string versionString); public abstract Task GetCodeFileAsync(string originalName, Stream stream, bool runAnalysis); + public virtual bool IsReviewGenByPipeline { get; } = false; + + public readonly CodeFileToken ReviewNotReadyCodeFile = new CodeFileToken("API review is being generated for this revision and it will be available in few minutes. Please refresh this page after few minutes to see generated API review.", CodeFileTokenKind.Literal); + public virtual CodeFile GetReviewGenPendingCodeFile(string fileName) => new CodeFile() + { + Name = fileName, + PackageName = fileName, + Language = Name, + Tokens = new CodeFileToken[] {new CodeFileToken("", CodeFileTokenKind.Newline), ReviewNotReadyCodeFile, new CodeFileToken("", CodeFileTokenKind.Newline) }, + Navigation = new NavigationItem[] { new NavigationItem() { Text = fileName } } + }; } } diff --git a/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs b/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs new file mode 100644 index 00000000000..aa07ae9b9f6 --- /dev/null +++ b/src/dotnet/APIView/APIViewWeb/Models/ReviewGenPipelineParamModel.cs @@ -0,0 +1,10 @@ +namespace APIViewWeb.Models +{ + public class ReviewGenPipelineParamModel + { + public string ReviewID { get; set; } + public string RevisionID { get; set; } + public string FileID { get; set; } + public string FileName { get; set; } + } +} diff --git a/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Review.cshtml.cs b/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Review.cshtml.cs index 3e9834a8f7a..ce6cc53996e 100644 --- a/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Review.cshtml.cs +++ b/src/dotnet/APIView/APIViewWeb/Pages/Assemblies/Review.cshtml.cs @@ -221,13 +221,6 @@ private int ComputeActiveConversations(CodeLine[] lines, ReviewCommentsModel com return activeThreads; } - public async Task OnPostRefreshModelAsync(string id) - { - await _manager.UpdateReviewAsync(User, id); - - return RedirectToPage(new { id = id }); - } - public async Task OnPostToggleClosedAsync(string id) { await _manager.ToggleIsClosedAsync(User, id); diff --git a/src/dotnet/APIView/APIViewWeb/Repositories/BlobOriginalsRepository.cs b/src/dotnet/APIView/APIViewWeb/Repositories/BlobOriginalsRepository.cs index cb555034751..1431cc07844 100644 --- a/src/dotnet/APIView/APIViewWeb/Repositories/BlobOriginalsRepository.cs +++ b/src/dotnet/APIView/APIViewWeb/Repositories/BlobOriginalsRepository.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.IO; using System.Threading.Tasks; using Azure.Storage.Blobs; @@ -12,6 +13,8 @@ public class BlobOriginalsRepository { private BlobContainerClient _container; + public string GetContainerUrl() => _container.Uri.ToString(); + public BlobOriginalsRepository(IConfiguration configuration) { var connectionString = configuration["Blob:ConnectionString"]; @@ -39,4 +42,4 @@ public async Task DeleteOriginalAsync(string codeFileId) await GetBlobClient(codeFileId).DeleteAsync(); } } -} \ No newline at end of file +} diff --git a/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs b/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs index 4b6915cf720..b154043fc86 100644 --- a/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs +++ b/src/dotnet/APIView/APIViewWeb/Repositories/DevopsArtifactRepository.cs @@ -1,10 +1,11 @@ -using Microsoft.Extensions.Configuration; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; using System.Text.Json; using System.Threading.Tasks; @@ -16,11 +17,15 @@ public class DevopsArtifactRepository private readonly IConfiguration _configuration; private readonly string _devopsAccessToken; + private readonly string _pipeline_run_rest; + private readonly string _hostUrl; - public DevopsArtifactRepository(IConfiguration configuration) + public DevopsArtifactRepository(IConfiguration configuration, IHttpContextAccessor httpContextAccessor) { _configuration = configuration; _devopsAccessToken = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _configuration["Azure-Devops-PAT"]))); + _pipeline_run_rest = _configuration["Azure-Devops-Run-Ripeline-Rest"]; + _hostUrl = _configuration["APIVIew-Host-Url"]; } public async Task DownloadPackageArtifact(string repoName, string buildId, string artifactName, string filePath, string project, string format= "file") @@ -28,11 +33,15 @@ public async Task DownloadPackageArtifact(string repoName, string buildI var downloadUrl = await GetDownloadArtifactUrl(repoName, buildId, artifactName, project); if (!string.IsNullOrEmpty(downloadUrl)) { - if (!filePath.StartsWith("/")) + if(!string.IsNullOrEmpty(filePath)) { - filePath = "/" + filePath; + if (!filePath.StartsWith("/")) + { + filePath = "/" + filePath; + } + downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath; } - downloadUrl = downloadUrl.Split("?")[0] + "?format=" + format + "&subPath=" + filePath; + SetDevopsClientHeaders(); var downloadResp = await _devopsClient.GetAsync(downloadUrl); downloadResp.EnsureSuccessStatusCode(); @@ -71,5 +80,16 @@ private string GetArtifactRestAPIForRepo(string repoName) } return downloadArtifactRestApi; } + + public async Task RunPipeline(string pipelineName, string reviewDetails, string originalStorageUrl) + { + SetDevopsClientHeaders(); + //Create dictionary of all required parametes to run tools - generate--apireview pipeline in azure devops + var reviewDetailsDict = new Dictionary { { "Reviews", reviewDetails }, { "APIViewUrl", _hostUrl }, { "StorageContainerUrl", originalStorageUrl } }; + var pipelineParams = new Dictionary> { { "templateParameters", reviewDetailsDict } }; + var stringContent = new StringContent(JsonSerializer.Serialize(pipelineParams), Encoding.UTF8, "application/json"); + var response = await _devopsClient.PostAsync(_pipeline_run_rest, stringContent); + response.EnsureSuccessStatusCode(); + } } } diff --git a/src/dotnet/APIView/APIViewWeb/Repositories/ReviewManager.cs b/src/dotnet/APIView/APIViewWeb/Repositories/ReviewManager.cs index e65a0251af8..fa64c84736a 100644 --- a/src/dotnet/APIView/APIViewWeb/Repositories/ReviewManager.cs +++ b/src/dotnet/APIView/APIViewWeb/Repositories/ReviewManager.cs @@ -17,6 +17,7 @@ using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.AspNetCore.Authorization; +using Microsoft.Extensions.Configuration; namespace APIViewWeb.Repositories { @@ -51,8 +52,9 @@ public ReviewManager( CosmosCommentsRepository commentsRepository, IEnumerable languageServices, NotificationManager notificationManager, - DevopsArtifactRepository devopsArtifactRepository, - PackageNameManager packageNameManager) + DevopsArtifactRepository devopsClient, + PackageNameManager packageNameManager, + IConfiguration configuration) { _authorizationService = authorizationService; _reviewsRepository = reviewsRepository; @@ -61,7 +63,7 @@ public ReviewManager( _commentsRepository = commentsRepository; _languageServices = languageServices; _notificationManager = notificationManager; - _devopsArtifactRepository = devopsArtifactRepository; + _devopsArtifactRepository = devopsClient; _packageNameManager = packageNameManager; } @@ -188,31 +190,32 @@ private async Task UpdateReviewAsync(ReviewModel review) var fileOriginal = await _originalsRepository.GetOriginalAsync(file.ReviewFileId); var languageService = GetLanguageService(file.Language); if (languageService == null) - continue; + continue; // file.Name property has been repurposed to store package name and version string // This is causing issue when updating review using latest parser since it expects Name field as file name // We have added a new property FileName which is only set for new reviews // All older reviews needs to be handled by checking review name field var fileName = file.FileName ?? (Path.HasExtension(review.Name) ? review.Name : file.Name); - var codeFile = await languageService.GetCodeFileAsync(fileName, fileOriginal, review.RunAnalysis); - await _codeFileRepository.UpsertCodeFileAsync(revision.RevisionId, file.ReviewFileId, codeFile); - // update only version string - file.VersionString = codeFile.VersionString; + if (languageService.IsReviewGenByPipeline) + { + GenerateReviewOffline(review, revision.RevisionId, file.ReviewFileId, fileName); + } + else + { + var codeFile = await languageService.GetCodeFileAsync(fileName, fileOriginal, review.RunAnalysis); + await _codeFileRepository.UpsertCodeFileAsync(revision.RevisionId, file.ReviewFileId, codeFile); + // update only version string + file.VersionString = codeFile.VersionString; + await _reviewsRepository.UpsertReviewAsync(review); + } } catch (Exception ex) { _telemetryClient.TrackTrace("Failed to update review " + review.ReviewId); _telemetryClient.TrackException(ex); } } - } - await _reviewsRepository.UpsertReviewAsync(review); - } - - internal async Task UpdateReviewAsync(ClaimsPrincipal user, string id) - { - var review = await GetReviewAsync(user, id); - await UpdateReviewAsync(review); + } } public async Task AddRevisionAsync( @@ -255,9 +258,16 @@ private async Task AddRevisionAsync( review.ServiceName = p?.ServiceName ?? review.ServiceName; } + var languageService = _languageServices.Single(s => s.IsSupportedFile(name)); + //Run pipeline to generateteh review if sandbox is enabled + if (languageService.IsReviewGenByPipeline) + { + // Run offline review gen for review and reviewCodeFileModel + GenerateReviewOffline(review, revision.RevisionId, codeFile.ReviewFileId, name); + } + // auto subscribe revision creation user await _notificationManager.SubscribeAsync(review, user); - await _reviewsRepository.UpsertReviewAsync(review); await _notificationManager.NotifySubscribersOnNewRevisionAsync(revision, user); } @@ -271,7 +281,7 @@ private async Task CreateFileAsync( using var memoryStream = new MemoryStream(); var codeFile = await CreateCodeFile(originalName, fileStream, runAnalysis, memoryStream); var reviewCodeFileModel = await CreateReviewCodeFileModel(revisionId, memoryStream, codeFile); - reviewCodeFileModel.FileName = originalName; + reviewCodeFileModel.FileName = originalName; return reviewCodeFileModel; } @@ -284,12 +294,18 @@ public async Task CreateCodeFile( var languageService = _languageServices.FirstOrDefault(s => s.IsSupportedFile(originalName)); await fileStream.CopyToAsync(memoryStream); memoryStream.Position = 0; - - CodeFile codeFile = await languageService.GetCodeFileAsync( + CodeFile codeFile = null; + if (languageService.IsReviewGenByPipeline) + { + codeFile = languageService.GetReviewGenPendingCodeFile(originalName); + } + else + { + codeFile = await languageService.GetCodeFileAsync( originalName, memoryStream, runAnalysis); - + } return codeFile; } @@ -696,5 +712,63 @@ public async Task AutoArchiveReviews(int archiveAfterMonths) } } } + private void GenerateReviewOffline(ReviewModel review, string revisionId, string fileId, string fileName) + { + var param = new ReviewGenPipelineParamModel() + { + FileID = fileId, + ReviewID = review.ReviewId, + RevisionID = revisionId, + FileName = fileName + }; + var paramList = new List(); + paramList.Add(param); + var languageService = _languageServices.Single(s => s.Name == review.Language); + RunReviewGenPipeline(paramList, languageService.Name); + } + + public async Task UpdateReviewCodeFiles(string repoName, string buildId, string artifact, string project) + { + var stream = await _devopsArtifactRepository.DownloadPackageArtifact(repoName, buildId, artifact, filePath: null, project: project, format: "zip"); + var archive = new ZipArchive(stream); + foreach (var entry in archive.Entries) + { + var reviewFilePath = entry.FullName; + var reviewDetails = reviewFilePath.Split("/"); + + if (reviewDetails.Length < 4 || !reviewFilePath.EndsWith(".json")) + continue; + + var reviewId = reviewDetails[1]; + var revisionId = reviewDetails[2]; + var codeFile = await CodeFile.DeserializeAsync(entry.Open()); + + // Update code file with one downloaded from pipeline + var review = await _reviewsRepository.GetReviewAsync(reviewId); + if (review != null) + { + var revision = review.Revisions.SingleOrDefault(review => review.RevisionId == revisionId); + if (revision != null) + { + await _codeFileRepository.UpsertCodeFileAsync(revisionId, revision.SingleFile.ReviewFileId, codeFile); + revision.Files.FirstOrDefault().VersionString = codeFile.VersionString; + await _reviewsRepository.UpsertReviewAsync(review); + } + } + } + } + private async void RunReviewGenPipeline(List reviewGenParams, string language) + { + var jsonSerializerOptions = new JsonSerializerOptions() + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip + }; + var reviewParamString = JsonSerializer.Serialize(reviewGenParams, jsonSerializerOptions); + reviewParamString = reviewParamString.Replace("\"", "'"); + await _devopsArtifactRepository.RunPipeline($"tools - generate-{language}-apireview", + reviewParamString, + _originalsRepository.GetContainerUrl()); + } } }