Skip to content

Commit

Permalink
feat: Adds generators project and AsyncPaginationExtension generator (o…
Browse files Browse the repository at this point in the history
  • Loading branch information
nickfloyd authored Aug 11, 2022
1 parent 5386257 commit 1e3fa1d
Show file tree
Hide file tree
Showing 11 changed files with 1,085 additions and 880 deletions.
1 change: 1 addition & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/dotnet
# "install" the dotnet 3.1 & 5.0 runtime for tests
COPY --from=mcr.microsoft.com/dotnet/sdk:3.1 /usr/share/dotnet/shared /usr/share/dotnet/shared
COPY --from=mcr.microsoft.com/dotnet/sdk:5.0 /usr/share/dotnet/shared /usr/share/dotnet/shared
COPY --from=mcr.microsoft.com/dotnet/sdk:6.0 /usr/share/dotnet/shared /usr/share/dotnet/shared

# # Add mkdocs for doc generation
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get -y install --no-install-recommends python3-pip
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/netcore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: 5.0.*
- name: Setup .NET 6
uses: actions/setup-dotnet@v1
with:
dotnet-version: 6.0.*
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: 5.0.*

- name: Setup .NET 6
uses: actions/setup-dotnet@v1
with:
dotnet-version: 6.0.*
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:
Expand Down
10 changes: 10 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Run Generator",
"type": "coreclr",
"request": "launch",
"program": "${workspaceFolder}/Octokit.Generators/bin/Debug/net6.0/Octokit.Generators",
"args": [],
"cwd": "${workspaceFolder}/Octokit.Generators",
"console": "internalConsole",
"stopAtEntry": false
},
{
"name": "Run unit tests",
"type": "coreclr",
Expand Down
1,684 changes: 818 additions & 866 deletions Octokit.AsyncPaginationExtension/Extensions.cs

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,23 @@
<PackageTags>GitHub API Octokit linqpad-samples dotnetcore</PackageTags>
<Copyright>Copyright GitHub 2017</Copyright>
<LangVersion>9</LangVersion>
<Nullable>enable</Nullable>
<Nullable>enable</Nullable>
</PropertyGroup>

<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
<NetStandardImplicitPackageVersion>2.0.0</NetStandardImplicitPackageVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
<NetStandardImplicitPackageVersion>2.0.0</NetStandardImplicitPackageVersion>
</PropertyGroup>

<ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.2" PrivateAssets="All" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.2" PrivateAssets="All" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Octokit\Octokit.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Octokit\Octokit.csproj" />
</ItemGroup>

</Project>
119 changes: 119 additions & 0 deletions Octokit.Generators/AsyncPaginationExtensionsGenerator.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
using System.Linq;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Data;
using System.Text;

namespace Octokit.Generators
{

/// <summary>
/// AsyncPaginationExtensionsGenerator for generating pagination extensions for Octokit.net Clients that return collections.
/// </summary>
/// <remarks>
/// This generator originally appeared in https://github.com/octokit/octokit.net/pull/2516
/// The generator solves a small part of a larger effort that is being discussed:
/// https://github.com/octokit/octokit.net/discussions/2499
/// https://github.com/octokit/octokit.net/discussions/2495
/// https://github.com/octokit/octokit.net/issues/2517
/// In the future, we should be able to unify generation for
/// * models (request and response)
/// * clients
/// * routing and related helpers
/// TODO: Convert to use Rosyln source generators
/// </remarks>
class AsyncPaginationExtensionsGenerator
{

private const string HEADER = (
@"using System;
using System.Collections.Generic;
namespace Octokit.AsyncPaginationExtension
{
/// <summary>
/// Provides all extensions for pagination.
/// </summary>
/// <remarks>
/// The <code>pageSize</code> parameter at the end of all methods allows for specifying the amount of elements to be fetched per page.
/// Only useful to optimize the amount of API calls made.
/// </remarks>
public static class Extensions
{
private const int DEFAULT_PAGE_SIZE = 30;
");

private const string FOOTER = (
@"
}
}");

/// <summary>
/// GenerateAsync static entry point for generating pagination extensions.
/// </summary>
/// <remarks>
/// This defaults the search path to the root of the project
/// This expects to generate the resulting code and put it in Octokit.AsyncPaginationExtension
/// This does a wholesale overwrite on ./Octokit.AsyncPaginationExtension/Extensions.cs
/// </remarks>
public static async Task GenerateAsync(string root = "./")
{
var sb = new StringBuilder(HEADER);
var enumOptions = new EnumerationOptions { RecurseSubdirectories = true };
var paginatedCallRegex = new Regex(@".*Task<IReadOnlyList<(?<returnType>\w+)>>\s*(?<name>\w+)(?<template><.*>)?\((?<arg>.*?)(, )?ApiOptions \w*\);");

foreach (var file in Directory.EnumerateFiles(root, "I*.cs", enumOptions)) {
var type = Path.GetFileNameWithoutExtension(file);

foreach (var line in File.ReadAllLines(file)) {
var match = paginatedCallRegex.Match(line);

if (!match.Success) { continue; }
sb.Append(BuildBodyFromTemplate(match, type));
}
}

sb.Append(FOOTER);

await File.WriteAllTextAsync("./Octokit.AsyncPaginationExtension/Extensions.cs", sb.ToString());
}

/// <summary>
/// BuildBodyFromTemplate uses the match from the regex search and parses values from the given source
/// to use to generate the paging implementations.
/// </summary>
/// <remarks>
/// TODO: This should be reworked to use source templates
/// </remarks>
private static string BuildBodyFromTemplate(Match match, string type)
{
var argSplitRegex = new Regex(@" (?![^<]*>)");
var returnType = match.Groups["returnType"].Value;
var name = match.Groups["name"].Value;
var arg = match.Groups["arg"].Value;
var template = match.Groups["template"];
var templateStr = template.Success ? template.Value : string.Empty;
var splitArgs = argSplitRegex.Split(arg).ToArray();

var lambda = arg.Length == 0
? $"t.{name}{templateStr}"
: $"options => t.{name}{templateStr}({string.Join(' ', splitArgs.Where((_, i) => i % 2 == 1))}, options)";

var docArgs = string.Join(", ", splitArgs.Where((_, i) => i % 2 == 0)).Replace('<', '{').Replace('>', '}');
if (docArgs.Length != 0) {
docArgs += ", ";
}

if (arg.Length != 0) {
arg += ", ";
}

return ($@"
/// <inheritdoc cref=""{type}.{name}({docArgs}ApiOptions)""/>
public static IPaginatedList<{returnType}> {name}Async{templateStr}(this {type} t, {arg}int pageSize = DEFAULT_PAGE_SIZE)
=> pageSize > 0 ? new PaginatedList<{returnType}>({lambda}, pageSize) : throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, ""The page size must be positive."");
");
}
}
}
29 changes: 29 additions & 0 deletions Octokit.Generators/Generator.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
using System.Threading.Tasks;

namespace Octokit.Generators
{
/// <summary>
/// Provides an entry point for code generation of various types.
/// </summary>
/// <remarks>
/// The backing source for generation will either be the source files in this repo or
/// the OpenAPI Descriptions from the GitHub REST API: https://github.com/github/rest-api-description
/// </remarks>
class Generator
{

static void Main(string[] args)
{

var operation = args.Length != 0 ? args[0] : "AsyncPaginationExtensions";

if (operation == "AsyncPaginationExtensions")
{
Task task = Task.Run( () => AsyncPaginationExtensionsGenerator.GenerateAsync());
task.Wait();
}

// Put more generation operations here, convert to case when needed.
}
}
}
23 changes: 23 additions & 0 deletions Octokit.Generators/Octokit.Generators.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>A set of code generators for Octokit.NET backed by the GitHub REST API Open API descriptions</Description>
<AssemblyTitle>Octokit.Generators</AssemblyTitle>
<Authors>GitHub</Authors>
<Version>0.0.0-dev</Version>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<AssemblyName>Octokit.Generators</AssemblyName>
<PackageId>Octokit.Generators</PackageId>
<DebugType>embedded</DebugType>
<RepositoryUrl>https://github.com/octokit/octokit.net</RepositoryUrl>
<PackageProjectUrl>https://github.com/octokit/octokit.net</PackageProjectUrl>
<PackageIconUrl>https://f.cloud.github.com/assets/19977/1510987/64af2b26-4a9d-11e3-89fc-96a185171c75.png</PackageIconUrl>
<PackageIcon>octokit.png</PackageIcon>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<PackageTags>GitHub API Octokit dotnetcore dotnetstandard2.0</PackageTags>
<Copyright>Copyright GitHub 2022</Copyright>
<LangVersion>9</LangVersion>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
58 changes: 58 additions & 0 deletions Octokit.Generators/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Octokit.net Generators

## About

We've been discussing and thinking about code generators for a [while now](https://github.com/octokit/octokit.net/discussions/2527) and they are part of the [future vision](https://github.com/octokit/octokit.net/discussions/2495) for where we'd like to head with this SDK.

Consider this to be iteration 0, meaning while we want to potentially move to source generators, templates, and using the features that Rosyln offers we need to do some functional experiments and solve our current needs while iterating on the future. Acknowledging that code generation is a solved problem (menaing there is existing work out there) we should be intentional with the direction we take.

----

## Getting started

From the Octokit .NET root run:

`dotnet run --project Octokit.Generators`

----

## Debugging

There is a launch config defined for this project named `Run Generator`

----

## CI/Actions

Currently no generation is automatically run as a build/release step. Once the vision solidifies here a bit we'll begin introducing automation so that whatever is generated is always up to date.

----

## Notes and thoughts on code generation for Octokit.net

### Code generation, interpreters, and language interpolation

Hoisted from this [discussion](https://github.com/octokit/octokit.net/discussions/2495)

As you know there are loads of things that we can do here - especially given the power of .NET, reflection, and Rosyln. Just two thoughts on the cautious side of things here:

1. Just because we can generate things, it does not mean we should. As we roll through discovery on all of this we might find out some areas where generation would be trying to put a square peg in a round hole. We need to have the courage to stand down when needed.
2. Our long-term goal should be language and platform independence. Meaning, we might nail down incredible generative aspects for the .NET SDK but we should always be targeting things like versioned, package distributed, models that the core SDK can reference as well as a generative engine that could potentially generate the models and implementations based on language templates, RFCs, and the like for any reference language - so generating models and SDK methods should be doable for both .NET and Go (for instance) using the same "engine" if possible. It's lofty but I feel that it could be possible.

So what might the future look like for code generation given the above statements? Let's have a look a what might be a really naive/potential roadmap (again we need the community's input and involvement here - which is why I am grateful for the questions):

1. Make sure our current solution addresses the community needs: All SDKs have been synchronized with the current API surface
2. Standardize implementations - testing, models, API methods, etc... (this is a critical step to get us to generation)
3. As we go through the above we will learn more about what works and what doesn't. Armed with that knowledge, begin prototyping generative models for each SDK
4. Solve OpenAPI generation for at least 1 SDK and implement - again with our sights on other languages
5. Publish (but don't reference yet) SDK models to packaging platforms (in this case nuget)
6. Work on Generative API implementations - methods etc..
7. Model generation is unified in the SDK (i.e. we used the published / versioned models in the SDK) and published to package platforms.


After this point, things get really interesting with things like:

- Recipe and workflow generation - think of things like plugins for the SDKs to do things like instantiating project boards with teams and referenced repos all in one SDK call. SDKs shouldn't be reflective API implementations, but rather tools that simplify our approach to getting things done.
- We could start generating SDKs using language specifications - imagine pairing our OpenAPI specification with a language RFC to spit out a baseline SDK that could be extended.
- Generating language agnostic SDKs - what if we could maximize code reuse by implementing interpreted recipes or code
- Building workflow interpolation based on user patterns present in GitHub - this comes back to the community and GitHub joining up to make consistent workflows no matter where the users are - in community apps or GitHub proper.
6 changes: 6 additions & 0 deletions Octokit.sln
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{64FD6CD6
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octokit.AsyncPaginationExtension", "Octokit.AsyncPaginationExtension\Octokit.AsyncPaginationExtension.csproj", "{0E8013E0-0CCF-4433-9E01-51AC288824C5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Octokit.Generators", "Octokit.Generators\Octokit.Generators.csproj", "{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -47,6 +49,10 @@ Global
{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E8013E0-0CCF-4433-9E01-51AC288824C5}.Release|Any CPU.Build.0 = Release|Any CPU
{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A577AD0-2319-4FFC-8EF2-3CCDD31B0CD9}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down

0 comments on commit 1e3fa1d

Please sign in to comment.