CromwellOnAzure On AKS

src/Common/Common.csproj

@@ -4,4 +4,12 @@
     <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Azure.Core" Version="1.22.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Autorest\" />
+  </ItemGroup>
+
 </Project>

src/deploy-cromwell-on-azure/Configuration.cs

@@ -29,6 +29,8 @@ public class Configuration
         public string LogAnalyticsArmId { get; set; }
         public string ApplicationInsightsAccountName { get; set; }
         public string VmName { get; set; }
+        public bool UseAks { get; set; }
+        public string AksCluserName { get; set; }
         public bool Silent { get; set; }
         public bool DeleteResourceGroupOnFailure { get; set; }
         public string CromwellVersion { get; set; }

src/deploy-cromwell-on-azure/Utility.cs

@@ -21,6 +21,12 @@ public static Dictionary<string, string> DelimitedTextToDictionary(string text,
         public static string DictionaryToDelimitedText(Dictionary<string, string> dictionary, string fieldDelimiter = "=", string rowDelimiter = "\n")
             => string.Join(rowDelimiter, dictionary.Select(kv => $"{kv.Key}{fieldDelimiter}{kv.Value}"));
 
+        public static Stream GetFileStream(params string[] pathComponentsRelativeToAppBase)
+        {
+            var embeddedResourceName = $"deploy-cromwell-on-azure.{string.Join(".", pathComponentsRelativeToAppBase)}";
+            return typeof(Deployer).Assembly.GetManifestResourceStream(embeddedResourceName);
+        }
+
         public static string GetFileContent(params string[] pathComponentsRelativeToAppBase)
         {
             var embeddedResourceName = $"deploy-cromwell-on-azure.{string.Join(".", pathComponentsRelativeToAppBase)}";

src/deploy-cromwell-on-azure/deploy-cromwell-on-azure.csproj

@@ -17,12 +17,35 @@
     <AssemblyName Condition="'$(Configuration)' == 'osx'">deploy-cromwell-on-azure-osx.app</AssemblyName>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Remove="scripts\k8s\cromwell-deployment.yaml" />
+    <None Remove="scripts\k8s\cromwell-service.yaml" />
+    <None Remove="scripts\k8s\cromwell-tmp-claim2.yaml" />
+    <None Remove="scripts\k8s\cromwellazure-claim1.yaml" />
+    <None Remove="scripts\k8s\csi\blobfuse-proxy.yaml" />
+    <None Remove="scripts\k8s\csi\csi-blob-controller.yaml" />
+    <None Remove="scripts\k8s\csi\csi-blob-driver.yaml" />
+    <None Remove="scripts\k8s\csi\csi-blob-node.yaml" />
+    <None Remove="scripts\k8s\csi\pv-blobfuse-csi-dataset.yaml" />
+    <None Remove="scripts\k8s\csi\pvc-blob-csi-dataset.yaml" />
+    <None Remove="scripts\k8s\csi\rbac-csi-blob-controller.yaml" />
+    <None Remove="scripts\k8s\csi\rbac-csi-blob-node.yaml" />
+    <None Remove="scripts\k8s\mysqldb-data-claim3.yaml" />
+    <None Remove="scripts\k8s\mysqldb-deployment.yaml" />
+    <None Remove="scripts\k8s\mysqldb-service.yaml" />
+    <None Remove="scripts\k8s\tes-deployment.yaml" />
+    <None Remove="scripts\k8s\tes-service.yaml" />
+    <None Remove="scripts\k8s\triggerservice-deployment.yaml" />
+  </ItemGroup>
+
   <ItemGroup>
     <Compile Include="..\CommonAssemblyInfo.cs" Link="Properties\CommonAssemblyInfo.cs" />
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="KubernetesClient" Version="7.0.9" />
     <PackageReference Include="Microsoft.Azure.Management.Batch" Version="13.0.0" />
+    <PackageReference Include="Microsoft.Azure.Management.ContainerService" Version="1.2.0" />
     <PackageReference Include="Microsoft.Azure.Management.Fluent" Version="1.37.1" />
     <PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.6.1" />
     <PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.2.3" />
@@ -37,8 +60,9 @@
     <PackageReference Include="Microsoft.IdentityModel.Clients.ActiveDirectory" Version="5.2.9" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="Polly" Version="7.2.2" />
+    <PackageReference Include="SharpZipLib" Version="1.3.3" />
     <PackageReference Include="SSH.NET" Version="2020.0.1" />
-    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.10.2" />
+    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.13.1" />
   </ItemGroup>
 
   <ItemGroup>
@@ -63,6 +87,24 @@
     <EmbeddedResource Include="scripts\env-03-external-images.txt" />
     <EmbeddedResource Include="scripts\env-04-settings.txt" />
     <EmbeddedResource Include="scripts\install-cromwellazure.sh" />
+    <EmbeddedResource Include="scripts\k8s\cromwell-deployment.yaml" />
+    <EmbeddedResource Include="scripts\k8s\cromwell-service.yaml" />
+    <EmbeddedResource Include="scripts\k8s\cromwell-tmp-claim2.yaml" />
+    <EmbeddedResource Include="scripts\k8s\cromwellazure-claim1.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\blobfuse-proxy.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\csi-blob-controller.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\csi-blob-driver.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\csi-blob-node.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\pv-blobfuse-csi-dataset.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\pvc-blob-csi-dataset.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\rbac-csi-blob-controller.yaml" />
+    <EmbeddedResource Include="scripts\k8s\csi\rbac-csi-blob-node.yaml" />
+    <EmbeddedResource Include="scripts\k8s\mysqldb-data-claim3.yaml" />
+    <EmbeddedResource Include="scripts\k8s\mysqldb-deployment.yaml" />
+    <EmbeddedResource Include="scripts\k8s\mysqldb-service.yaml" />
+    <EmbeddedResource Include="scripts\k8s\tes-deployment.yaml" />
+    <EmbeddedResource Include="scripts\k8s\tes-service.yaml" />
+    <EmbeddedResource Include="scripts\k8s\triggerservice-deployment.yaml" />
     <EmbeddedResource Include="scripts\mount-data-disk.sh" />
     <EmbeddedResource Include="scripts\mount.blobfuse" />
     <EmbeddedResource Include="scripts\mount_containers.sh" />

src/deploy-cromwell-on-azure/scripts/k8s/cromwell-deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    kompose.cmd: C:\workspace\src\CromwellOnAzure\src\deploy-cromwell-on-azure\scripts\kompose.exe convert
+    kompose.version: 1.26.0 (40646f47)
+  creationTimestamp: null
+  labels:
+    io.kompose.service: cromwell
+  name: cromwell
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      io.kompose.service: cromwell
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      annotations:
+        kompose.cmd: C:\workspace\src\CromwellOnAzure\src\deploy-cromwell-on-azure\scripts\kompose.exe convert
+        kompose.version: 1.26.0 (40646f47)
+      creationTimestamp: null
+      labels:
+        io.kompose.service: cromwell
+    spec:
+      containers:
+        - args:
+            -  java -Djava.io.tmpdir=/cromwell-tmp/ -Dconfig.file=/configuration/cromwell-application.conf -jar /app/cromwell.jar server
+          command:
+            - /bin/sh
+            - -c
+          image: broadinstitute/cromwell:70
+          name: cromwell
+          stdin: true
+          tty: true
+          ports:
+            - containerPort: 8000
+          resources: {}
+          volumeMounts:
+            - mountPath: /cromwell-tmp
+              name: cromwell-tmp-claim2
+            - mountPath: /configuration
+              name: configuration-claim1
+            - mountPath: /cromwell-executions
+              name: cromwell-executions-claim1
+            - mountPath: /cromwell-workflow-logs
+              name: cromwell-workflow-logs-claim1
+            - mountPath: /workflows
+              name: workflows-claim1
+            - mountPath: /inputs
+              name: inputs-claim1
+            - mountPath: /outputs
+              name: outputs-claim1
+            - mountPath: /datasettestinputs/dataset
+              name: dataset-claim1
+      restartPolicy: Always
+      volumes:
+        - name: cromwell-tmp-claim2
+          persistentVolumeClaim:
+            claimName: cromwell-tmp-claim2
+        - name: configuration-claim1
+          persistentVolumeClaim:
+            claimName: configuration-claim1
+        - name: cromwell-executions-claim1
+          persistentVolumeClaim:
+            claimName: cromwell-executions-claim1
+        - name: cromwell-workflow-logs-claim1
+          persistentVolumeClaim:
+            claimName: cromwell-workflow-logs-claim1
+        - name: workflows-claim1
+          persistentVolumeClaim:
+            claimName: workflows-claim1
+        - name: inputs-claim1
+          persistentVolumeClaim:
+            claimName: inputs-claim1
+        - name: outputs-claim1
+          persistentVolumeClaim:
+            claimName: outputs-claim1
+        - name: dataset-claim1
+          persistentVolumeClaim:
+            claimName: pvc-blob-dataset
+status: {}

src/deploy-cromwell-on-azure/scripts/k8s/cromwell-service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    kompose.cmd: C:\workspace\src\CromwellOnAzure\src\deploy-cromwell-on-azure\scripts\kompose.exe convert
+    kompose.version: 1.26.0 (40646f47)
+  creationTimestamp: null
+  labels:
+    io.kompose.service: cromwell
+  name: cromwell
+spec:
+  ports:
+    - name: "8000"
+      port: 8000
+      targetPort: 8000
+  selector:
+    io.kompose.service: cromwell
+status:
+  loadBalancer: {}

src/deploy-cromwell-on-azure/scripts/k8s/cromwell-tmp-claim2.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: cromwell-tmp-claim2
+  name: cromwell-tmp-claim2
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 32Gi
+status: {}

src/deploy-cromwell-on-azure/scripts/k8s/cromwellazure-claim1.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  creationTimestamp: null
+  labels:
+    io.kompose.service: cromwellazure-claim1
+  name: cromwellazure-claim1
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 32Gi
+status: {}

src/deploy-cromwell-on-azure/scripts/k8s/csi/blobfuse-proxy.yaml

@@ -0,0 +1,115 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: csi-blobfuse-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-blobfuse-proxy
+  template:
+    metadata:
+      labels:
+        app: csi-blobfuse-proxy
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: type
+                    operator: NotIn
+                    values:
+                      - virtual-kubelet
+      initContainers:
+        - name: prepare-binaries
+          image: mcr.microsoft.com/k8s/csi/blob-csi:v1.8.0
+          command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"]
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp-dir
+      containers:
+        - command:
+            - nsenter
+            - '--target'
+            - '1'
+            - '--mount'
+            - '--uts'
+            - '--ipc'
+            - '--net'
+            - '--pid'
+            - '--'
+            - sh
+            - '-c'
+            - |
+              set -xe
+              INSTALL_BLOBFUSE=${INSTALL_BLOBFUSE:-true}
+              if (( "${INSTALL_BLOBFUSE}" == "true" ))
+              then
+                dpkg -i /tmp/packages-microsoft-prod.deb && apt update && apt-get install -y blobfuse=${BLOBFUSE_VERSION}
+              fi
+              dpkg -i /tmp/blobfuse-proxy.deb
+              mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
+              systemctl enable blobfuse-proxy
+              systemctl start blobfuse-proxy
+
+              SET_MAX_FILE_NUM=${SET_MAX_OPEN_FILE_NUM:-true}
+              if (( "${SET_MAX_OPEN_FILE_NUM}" == "true" ))
+              then
+                sysctl -w fs.file-max=${MAX_FILE_NUM}
+              fi
+
+              updateDBConfigPath="/etc/updatedb.conf"
+              DISABLE_UPDATEDB=${DISABLE_UPDATEDB:-true}
+              if (( "${DISABLE_UPDATEDB}" == "true" )) && (( test -f ${updateDBConfigPath} ))
+              then
+                echo "before changing ${updateDBConfigPath}:"
+                cat ${updateDBConfigPath}
+                sed -i 's/PRUNEPATHS="\/tmp/PRUNEPATHS="\/mnt \/var\/lib\/kubelet \/tmp/g' ${updateDBConfigPath}
+                sed -i 's/PRUNEFS="NFS/PRUNEFS="fuse blobfuse NFS/g' ${updateDBConfigPath}
+                echo "after change:"
+                cat ${updateDBConfigPath}
+              fi
+
+              # "waiting for blobfuse-proxy service to start"
+              sleep 3s
+              # tail blobfuse proxy logs
+              journalctl -u blobfuse-proxy -f
+          image: mcr.microsoft.com/k8s/csi/blob-csi:v1.8.0
+          imagePullPolicy: IfNotPresent
+          name: sysctl-install-blobfuse-proxy
+          env:
+            - name: DEBIAN_FRONTEND
+              value: "noninteractive"
+            - name: INSTALL_BLOBFUSE
+              value: "true"
+            - name: BLOBFUSE_VERSION
+              value: 1.4.2
+            - name: SET_MAX_OPEN_FILE_NUM
+              value: "true"
+            - name: MAX_FILE_NUM
+              value: "9000000"
+            - name: DISABLE_UPDATEDB
+              value: "true"
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 1Mi
+          securityContext:
+            privileged: true
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      tolerations:
+        - operator: Exists
+      volumes:
+        - hostPath:
+            path: /tmp
+            type: DirectoryOrCreate
+          name: tmp-dir