Skip to content

Commit

Permalink
feat!: enable horizontal autoscaling of Liquid Legions v2 Mill
Browse files Browse the repository at this point in the history
  • Loading branch information
@SanjayVas
SanjayVas committed Jul 22, 2024
1 parent d8385d2 commit 6e0ae9c
Show file tree
Hide file tree
Showing 34 changed files with 1,015 additions and 366 deletions.
17 changes: 11 additions & 6 deletions src/main/docker/images.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ COMMON_IMAGES = [
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job:computations_cleaner_image",
repository = _PREFIX + "/duchy/computations-cleaner",
),
struct(
name = "duchy_mill_job_scheduler_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill:job_scheduler_image",
repository = _PREFIX + "/duchy/mill-job-scheduler",
),
struct(
name = "kingdom_data_server_image",
image = "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server:gcp_kingdom_data_server_image",
Expand Down Expand Up @@ -127,8 +132,8 @@ GKE_IMAGES = [
repository = _PREFIX + "/duchy/requisition-fulfillment",
),
struct(
name = "duchy_liquid_legions_v2_mill_daemon_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2:gcs_liquid_legions_v2_mill_daemon_image",
name = "duchy_liquid_legions_v2_mill_job_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/job/mill/liquidlegionsv2:gcs_liquid_legions_v2_mill_job_image",
repository = _PREFIX + "/duchy/liquid-legions-v2-mill",
),
struct(
Expand Down Expand Up @@ -172,8 +177,8 @@ EKS_IMAGES = [
repository = _PREFIX + "/duchy/aws-requisition-fulfillment",
),
struct(
name = "duchy_s3_liquid_legions_v2_mill_daemon_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/liquidlegionsv2:s3_liquid_legions_v2_mill_daemon_image",
name = "duchy_s3_liquid_legions_v2_mill_job_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/job/mill/liquidlegionsv2:s3_liquid_legions_v2_mill_job_image",
repository = _PREFIX + "/duchy/aws-liquid-legions-v2-mill",
),
struct(
Expand All @@ -196,8 +201,8 @@ LOCAL_IMAGES = [
repository = _PREFIX + "/duchy/local-herald",
),
struct(
name = "forwarded_storage_liquid_legions_v2_mill_daemon_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2:forwarded_storage_liquid_legions_v2_mill_daemon_image",
name = "forwarded_storage_liquid_legions_v2_mill_job_image",
image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/job/mill/liquidlegionsv2:forwarded_storage_liquid_legions_v2_mill_job_image",
repository = _PREFIX + "/duchy/local-liquid-legions-v2-mill",
),
struct(
Expand Down
59 changes: 48 additions & 11 deletions src/main/k8s/base.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,49 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
tolerations: [ for _, toleration in _tolerations {toleration}]
}

// K8s PodTemplateSpec.
#PodTemplateSpec: {
metadata: #ObjectMeta & {
annotations: {
"instrumentation.opentelemetry.io/inject-java": string | *"true"
}
}
spec: #PodSpec
}

// K8s PodTemplate.
#PodTemplate: {
let Name = metadata.name

_secretName?: string
_container: #Container & {
_javaOptions: {
heapDumpOnOutOfMemory: true
heapDumpPath: "/run/heap-dumps"
}
}

apiVersion: "v1"
kind: "PodTemplate"
metadata: #ObjectMeta
template: #PodTemplateSpec & {
metadata: labels: {
app: "\(Name)-app"
}
spec: {
_mounts: {
if _secretName != _|_ {
"\(Name)-files": {
volume: secret: secretName: _secretName
}
}
"heap-dumps": volume: emptyDir: {}
}
_containers: "\(Name)-container": _container
}
}
}

// K8s Pod.
#Pod: {
apiVersion: "v1"
Expand Down Expand Up @@ -522,16 +565,13 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
selector: #LabelSelector & {
matchLabels: app: _name + "-app"
}
template: {
template: #PodTemplateSpec & {
metadata: {
labels: {
app: _name + "-app"
}
annotations: {
"instrumentation.opentelemetry.io/inject-java": string | *"true"
}
}
spec: #PodSpec & {
spec: {
_mounts: {
if _secretName != _|_ {
"\(_name)-files": {
Expand Down Expand Up @@ -583,16 +623,13 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
jobTemplate: {
spec: {
backoffLimit: uint | *0
template: {
metadata: #ObjectMeta & {
template: #PodTemplateSpec & {
metadata: {
labels: {
app: _name + "-app"
}
annotations: {
"instrumentation.opentelemetry.io/inject-java": _ | *"true"
}
}
spec: #PodSpec & {
spec: {
if _secretName != _|_ {
_mounts: "\(_name)-files": {
volume: secret: secretName: _secretName
Expand Down
31 changes: 17 additions & 14 deletions src/main/k8s/dev/duchy_eks.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)"
}
}
#Llv2MillMaxHeapSize: "1G"
#Llv2MillReplicas: 1
#Llv2MillMaxConcurrency: 10
#HmssMillResourceRequirements: ResourceRequirements=#ResourceRequirements & {
requests: {
cpu: "2"
Expand Down Expand Up @@ -95,7 +95,7 @@ duchy: #PostgresDuchy & {
_imageSuffixes: {
"herald-daemon": "duchy/aws-herald"
"computation-control-server": "duchy/aws-computation-control"
"liquid-legions-v2-mill-daemon": "duchy/aws-liquid-legions-v2-mill"
"llv2-mill": "duchy/aws-liquid-legions-v2-mill"
"hmss-mill-daemon": "duchy/aws-honest-majority-share-shuffle-mill"
"requisition-fulfillment-server": "duchy/aws-requisition-fulfillment"
"internal-api-server": "duchy/aws-postgres-internal-server"
Expand Down Expand Up @@ -132,18 +132,9 @@ duchy: #PostgresDuchy & {
serviceAccountName: #StorageServiceAccount
}
}
"liquid-legions-v2-mill-daemon-deployment": {
_workLockDuration: "10m"
_container: {
_javaOptions: maxHeapSize: #Llv2MillMaxHeapSize
resources: #Llv2MillResourceRequirements
}
spec: {
replicas: #Llv2MillReplicas
template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
serviceAccountName: #StorageServiceAccount
}
}
"mill-job-scheduler": {
_liquidLegionsV2MaxConcurrency: #Llv2MillMaxConcurrency
_liquidLegionsV2WorkLockDuration: "10m"
}
"hmss-mill-daemon-deployment": {
_workLockDuration: "5m"
Expand Down Expand Up @@ -185,4 +176,16 @@ duchy: #PostgresDuchy & {
}
}
}
podTemplates: {
"llv2-mill": {
_workLockDuration: "10m"
_container: {
_javaOptions: maxHeapSize: #Llv2MillMaxHeapSize
resources: #Llv2MillResourceRequirements
}
template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
serviceAccountName: #StorageServiceAccount
}
}
}
}
33 changes: 17 additions & 16 deletions src/main/k8s/dev/duchy_gke.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)"
}
}
#Llv2MillMaxHeapSize: "1G"
#Llv2MillReplicas: 1
#Llv2MillMaxConcurrency: 10
#HmssMillResourceRequirements: ResourceRequirements=#ResourceRequirements & {
requests: {
cpu: "2"
Expand Down Expand Up @@ -117,12 +117,11 @@ duchy: #SpannerDuchy & {
_verbose_grpc_logging: "false"
_duchyMillParallelism: 4

serviceAccounts: [string]: #WorkloadIdentityServiceAccount
serviceAccounts: {
"\(#InternalServerServiceAccount)": {
"\(#InternalServerServiceAccount)": #WorkloadIdentityServiceAccount & {
_iamServiceAccountName: "\(_duchy_name)-duchy-internal"
}
"\(#StorageServiceAccount)": {
"\(#StorageServiceAccount)": #WorkloadIdentityServiceAccount & {
_iamServiceAccountName: "\(_duchy_name)-duchy-storage"
}
}
Expand All @@ -147,18 +146,8 @@ duchy: #SpannerDuchy & {
serviceAccountName: #StorageServiceAccount
}
}
"liquid-legions-v2-mill-daemon-deployment": {
_workLockDuration: "10m"
_container: {
_javaOptions: maxHeapSize: #Llv2MillMaxHeapSize
resources: #Llv2MillResourceRequirements
}
spec: {
replicas: #Llv2MillReplicas
template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
serviceAccountName: #StorageServiceAccount
}
}
"mill-job-scheduler": {
_liquidLegionsV2MaxConcurrency: #Llv2MillMaxConcurrency
}
"hmss-mill-daemon-deployment": {
_workLockDuration: "5m"
Expand Down Expand Up @@ -197,4 +186,16 @@ duchy: #SpannerDuchy & {
"requisition-fulfillment-server": _ipAddressName: _publicApiAddressName
"computation-control-server": _ipAddressName: _systemApiAddressName
}

podTemplates: {
"llv2-mill": {
_container: {
_javaOptions: maxHeapSize: #Llv2MillMaxHeapSize
resources: #Llv2MillResourceRequirements
}
template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
serviceAccountName: #StorageServiceAccount
}
}
}
}
Loading

0 comments on commit 6e0ae9c

Please sign in to comment.