Add service to enable github issues workflow (#1)

* big bang gitub workflows

* add missing settings in local.settings.json

* config refactor

* fix devlead plan response

* swap cosmos to table storage for metadata storage

* unify config via options

* azd-ify WIP

* add qdrant bicep WIP

* working azd provision setup

* consolidate SK version in projects

* replace localhost :)

* add fqdn to options

* httpclient fixes

* add managed identity to the function and assign contrib role

* qdrant endpoint setting

* add container instances cleanup code + wait on termination to upload to Github

* formatting fixes

* add tables in bicep

* local getting started WIP

* add azure setup instructions

* add the load-waf bits

* docs WIP

---------

Co-authored-by: Kosta Petan <[email protected]>

.devcontainer/Dockerfile

@@ -1,5 +1,7 @@
 FROM mcr.microsoft.com/devcontainers/dotnet:0-7.0
 # Install the xz-utils package
-RUN apt-get update && apt-get install -y xz-utils nodejs
+RUN apt-get update && apt-get install -y xz-utils nodejs npm
 
-RUN curl -fsSL https://aka.ms/install-azd.sh | bash
+RUN curl -fsSL https://aka.ms/install-azd.sh | bash
+
+RUN npm i -g azure-functions-core-tools@4 --unsafe-perm true

.devcontainer/devcontainer.json

@@ -9,7 +9,8 @@
 	"features": {
 		"ghcr.io/devcontainers/features/azure-cli:1": {},
 		"ghcr.io/devcontainers/features/common-utils:2": {},
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/azure/azure-dev/azd:latest": {}
 	},
 	"postCreateCommand": "bash .devcontainer/startup.sh",
 	"hostRequirements": {
@@ -39,7 +40,13 @@
 				"ms-dotnettools.csdevkit",
 				"Azurite.azurite",
 				"ms-dotnettools.csharp",
-				"ms-semantic-kernel.semantic-kernel"
+				"ms-semantic-kernel.semantic-kernel",
+				"GitHub.copilot-chat",
+				"GitHub.vscode-github-actions",
+                "ms-azuretools.azure-dev",
+                "ms-azuretools.vscode-azurefunctions",
+                "ms-azuretools.vscode-bicep",
+                "ms-dotnettools.vscode-dotnet-runtime"
 			]
 		}
 	}

.gitignore

@@ -482,12 +482,17 @@ $RECYCLE.BIN/
 
 # Vim temporary swap files
 *.swp
-
+__azurite**
+__blob**
+__queue**
 # SQLite workflows DB
 elsa.sqlite.*
 
 # env files
 .env
 
 # ignore local elsa-core src
-elsa-core/
+elsa-core/
+sk-azfunc-server/local.settings.json
+.azure
+temp

azure.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json
+
+name: sk-dev-team
+services:
+  sk-func:
+    project: ./sk-azfunc-server
+    language: dotnet
+    host: function

cli/Models/DevLeadPlanResponse.cs

@@ -3,7 +3,7 @@
 public class Subtask
 {
     public string subtask { get; set; }
-    public string LLM_prompt { get; set; }
+    public string prompt { get; set; }
 }
 
 public class Step

cli/Program.cs

@@ -92,7 +92,7 @@ public static async Task ChainFunctions(string file, int maxRetry)
                     {
                         try
                         {
-                            implementationResult = await CallFunction<string>(nameof(Developer), Developer.Implement, subtask.LLM_prompt, maxRetry);
+                            implementationResult = await CallFunction<string>(nameof(Developer), Developer.Implement, subtask.prompt, maxRetry);
                             break;
                         }
                         catch (Exception ex)
@@ -137,7 +137,7 @@ public static async Task<T> CallFunction<T>(string skillName, string functionNam
                 .AddConsole()
                 .AddDebug();
         });
-        var memoryStore = new QdrantMemoryStore(new QdrantVectorDbClient("http://qdrant", 1536, port: 6333));
+        var memoryStore = new QdrantMemoryStore(new QdrantVectorDbClient("http://qdrant:6333", 1536));
         var embedingGeneration = new AzureTextEmbeddingGeneration(kernelSettings.EmbeddingDeploymentOrModelId, kernelSettings.Endpoint, kernelSettings.ApiKey);
         var semanticTextMemory = new SemanticTextMemory(memoryStore, embedingGeneration);
 

cli/SandboxSkill.cs

@@ -3,17 +3,11 @@
 
 public class SandboxSkill
 {
-    [SKFunction("Run a script in Alpine sandbox")]
-    [SKFunctionInput(Description = "The script to be executed")]
-    [SKFunctionName("RunInAlpine")]
     public async Task<string> RunInAlpineAsync(string input)
     {
         return await RunInContainer(input, "alpine");
     }
 
-    [SKFunction("Run a script in dotnet alpine sandbox")]
-    [SKFunctionInput(Description = "The script to be executed")]
-    [SKFunctionName("RunInDotnetAlpine")]
     public async Task<string> RunInDotnetAlpineAsync(string input)
     {
         return await RunInContainer(input, "mcr.microsoft.com/dotnet/sdk:7.0");

cli/cli.csproj

@@ -13,8 +13,8 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
-    <PackageReference Include="Microsoft.SemanticKernel" Version="0.15.230609.2-preview" />
-    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Memory.Qdrant" Version="0.15.230609.2-preview" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="0.18.230725.3-preview" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Memory.Qdrant" Version="0.18.230725.3-preview" />
     <PackageReference Include="Testcontainers" Version="3.2.0" />
     <ProjectReference Include="..\skills\skills.csproj" />
   </ItemGroup>

docs/github-flow-architecture.md

@@ -0,0 +1 @@
+# Azure components

docs/github-flow-getting-started.md

@@ -0,0 +1,90 @@
+## Prerequisites
+
+- Access to gpt3.5-turbo or preferably gpt4 - [Get access here](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview#how-do-i-get-access-to-azure-openai)
+- [Setup a Github app](#how-do-i-setup-the-github-app)
+- [Install the Github app](https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app)
+- [Create labels for the dev team skills](#which-labels-should-i-create)
+
+### How do I setup the Github app?
+
+- [Register a Github app](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app).
+- Setup the following permissions
+    - Repository 
+        - Contents - read and write
+        - Issues - read and write
+        - Metadata - read only
+        - Pull requests - read and write
+- Subscribe to the following events:
+    - Issues
+    - Issue comment
+- Allow this app to be installed by any user or organization
+- Add a dummy value for the webhook url, we'll come back to this setting
+- After the app is created, generate a private key, we'll use it later for authentication to Github from the app
+
+### Which labels should I create?
+
+In order for us to know which skill and persona we need to talk with, we are using Labels in Github Issues
+
+The default bunch of skills and personnas are as follows:
+- PM.Readme
+- PM.BootstrapProject
+- Do.It
+- DevLead.Plan
+- Developer.Implement
+
+Once you start adding your own skills, just remember to add the corresponding Label!
+
+## How do I run this locally?
+
+Codespaces are preset for this repo.
+
+Create a codespace and once the codespace is created, make sure to fill in the `local.settings.json` file.
+
+There is a `local.settings.template.json` you can copy and fill in, containing comments on the different config values.
+
+Hit F5 and go to the Ports tab in your codespace, make sure you make the `:7071` port publically visible. [How to share port?](https://docs.github.com/en/codespaces/developing-in-codespaces/forwarding-ports-in-your-codespace?tool=vscode#sharing-a-port-1)
+
+Copy the local address (it will look something like https://foo-bar-7071.preview.app.github.dev) and append `/api/github/webhooks` at the end. Using this value, update the Github App's webhook URL and you are ready to go!
+
+Before you go and have the best of times, there is one last thing left to do [load the WAF into the vector DB](#load-the-waf-into-qdrant)
+
+
+
+## How do I deploy this to Azure?
+
+This repo is setup to use  [azd](https://learn.microsoft.com/en-us/azure/developer/azure-developer-cli/overview) to work with the Azure bits. `azd` is installed in the codespace.
+
+Let's start by logging in to Azure using
+```bash
+azd auth login
+```
+
+After we've logged in, we need to create a new environment and setup the OpenAI and GithubApp config.
+
+```bash
+azd env new dev
+azd env set -e dev GH_APP_ID replace_with_gh_app_id
+azd env set -e dev GH_APP_INST_ID replace_with_inst_id
+azd env set -e dev GH_APP_KEY replace_with_gh_app_key
+azd env set -e dev OAI_DEPLOYMENT_ID replace_with_deployment_id
+azd env set -e dev OAI_EMBEDDING_ID replace_with_embedding_id
+azd env set -e dev OAI_ENDPOINT replace_with_oai_endpoint
+azd env set -e dev OAI_KEY replace_with_oai_key
+azd env set -e dev OAI_SERVICE_ID replace_with_oai_service_id
+azd env set -e dev OAI_SERVICE_TYPE AzureOpenAI
+```
+
+Now that we have all that setup, the only thing left to do is run
+
+```
+azd up -e dev
+```
+
+and wait for the azure components to be provisioned and the app deployed.
+
+As the last step, we also need to [load the WAF into the vector DB](#load-the-waf-into-qdrant)
+
+### Load the WAF into Qdrant. 
+
+If you are running the app locally, we have [Qdrant](https://qdrant.tech/) setup in the Codespace and if you are running in Azure, Qdrant is deployed to ACA.
+The loader is a project in the `util` folder, called `seed-memory`. We need to fill in the `appsettings.json` file in the `config` folder with the OpenAI details and the Qdrant endpoint, then just run the loader with `dotnet run` and you are ready to go.

docs/github-flow.md

@@ -0,0 +1 @@
+![](/docs/images/github-sk-dev-team.png)

infra/abbreviations.json

@@ -0,0 +1,11 @@
+{
+    "appManagedEnvironments": "cae-",
+    "containerRegistryRegistries": "cr",
+    "insightsComponents": "appi-",
+    "operationalInsightsWorkspaces": "log-",
+    "portalDashboards": "dash-",
+    "resourcesResourceGroups": "rg-",
+    "storageStorageAccounts": "st",
+    "webServerFarms": "plan-",
+    "webSitesFunctions": "func-"
+}

infra/app/sk-func.bicep

@@ -0,0 +1,45 @@
+param name string
+param location string = resourceGroup().location
+param tags object = {}
+
+param allowedOrigins array = []
+param applicationInsightsName string = ''
+param appServicePlanId string
+@secure()
+param appSettings object = {}
+param serviceName string = 'sk-func'
+param storageAccountName string
+
+module api '../core/host/functions.bicep' = {
+  name: '${serviceName}-functions-dotnet-isolated-module'
+  params: {
+    name: name
+    location: location
+    tags: union(tags, { 'azd-service-name': serviceName })
+    allowedOrigins: allowedOrigins
+    alwaysOn: false
+    appSettings: appSettings
+    applicationInsightsName: applicationInsightsName
+    appServicePlanId: appServicePlanId
+    runtimeName: 'dotnet-isolated'
+    runtimeVersion: '7.0'
+    storageAccountName: storageAccountName
+    scmDoBuildDuringDeployment: false
+    managedIdentity: true
+  }
+}
+
+var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')
+
+resource rgContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(subscription().id, resourceGroup().id, contributorRole)
+  properties: {
+    roleDefinitionId: contributorRole
+    principalType: 'ServicePrincipal'
+    principalId: api.outputs.identityPrincipalId
+  }
+}
+
+output SERVICE_API_IDENTITY_PRINCIPAL_ID string = api.outputs.identityPrincipalId
+output SERVICE_API_NAME string = api.outputs.name
+output SERVICE_API_URI string = api.outputs.uri

infra/core/database/postgresql/flexibleserver.bicep

@@ -0,0 +1,64 @@
+param name string
+param location string = resourceGroup().location
+param tags object = {}
+
+param sku object
+param storage object
+param administratorLogin string
+@secure()
+param administratorLoginPassword string
+param databaseNames array = []
+param allowAzureIPsFirewall bool = false
+param allowAllIPsFirewall bool = false
+param allowedSingleIPs array = []
+
+// PostgreSQL version
+param version string
+
+// Latest official version 2022-12-01 does not have Bicep types available
+resource postgresServer 'Microsoft.DBforPostgreSQL/flexibleServers@2022-12-01' = {
+  location: location
+  tags: tags
+  name: name
+  sku: sku
+  properties: {
+    version: version
+    administratorLogin: administratorLogin
+    administratorLoginPassword: administratorLoginPassword
+    storage: storage
+    highAvailability: {
+      mode: 'Disabled'
+    }
+  }
+
+  resource database 'databases' = [for name in databaseNames: {
+    name: name
+  }]
+
+  resource firewall_all 'firewallRules' = if (allowAllIPsFirewall) {
+    name: 'allow-all-IPs'
+    properties: {
+      startIpAddress: '0.0.0.0'
+      endIpAddress: '255.255.255.255'
+    }
+  }
+
+  resource firewall_azure 'firewallRules' = if (allowAzureIPsFirewall) {
+    name: 'allow-all-azure-internal-IPs'
+    properties: {
+      startIpAddress: '0.0.0.0'
+      endIpAddress: '0.0.0.0'
+    }
+  }
+
+  resource firewall_single 'firewallRules' = [for ip in allowedSingleIPs: {
+    name: 'allow-single-${replace(ip, '.', '')}'
+    properties: {
+      startIpAddress: ip
+      endIpAddress: ip
+    }
+  }]
+
+}
+
+output POSTGRES_DOMAIN_NAME string = postgresServer.properties.fullyQualifiedDomainName