Add executable binary classes and docker images for HMSS. (#1632)

world-federation-of-advertisers · Aug 16, 2024 · f297ec3 · f297ec3
1 parent 2114489
commit f297ec3
Show file tree

Hide file tree

Showing 50 changed files with 1,319 additions and 173 deletions.
diff --git a/.github/workflows/configure-aws-duchy.yml b/.github/workflows/configure-aws-duchy.yml
@@ -93,6 +93,7 @@ jobs:
         POSTGRES_HOST: ${{ vars.AWS_POSTGRES_HOST }}
         POSTGRES_CRED_SECRET_NAME: ${{ vars.AWS_POSTGRES_CRED_SECRET_NAME }}
         KINGDOM_SYSTEM_API_TARGET: ${{ vars.KINGDOM_SYSTEM_API_TARGET }}
+        KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }}
         AGGREGATOR_SYSTEM_API_TARGET: ${{ vars.AGGREGATOR_SYSTEM_API_TARGET }}
         WORKER1_SYSTEM_API_TARGET: ${{ vars.WORKER1_SYSTEM_API_TARGET }}
         WORKER2_SYSTEM_API_TARGET: ${{ vars.WORKER2_SYSTEM_API_TARGET }}
@@ -105,6 +106,7 @@ jobs:
         common --config=ghcr
         build --define image_tag=$IMAGE_TAG
         build --define kingdom_system_api_target=$KINGDOM_SYSTEM_API_TARGET
+        build --define kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET
         build --define aggregator_system_api_target=$AGGREGATOR_SYSTEM_API_TARGET
         build --define worker1_system_api_target=$WORKER1_SYSTEM_API_TARGET
         build --define worker2_system_api_target=$WORKER2_SYSTEM_API_TARGET

diff --git a/.github/workflows/configure-duchy.yml b/.github/workflows/configure-duchy.yml
@@ -91,6 +91,7 @@ jobs:
       env:
         IMAGE_TAG: ${{ inputs.image-tag }}
         KINGDOM_SYSTEM_API_TARGET: ${{ vars.KINGDOM_SYSTEM_API_TARGET }}
+        KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }}
         AGGREGATOR_SYSTEM_API_TARGET: ${{ vars.AGGREGATOR_SYSTEM_API_TARGET }}
         WORKER1_SYSTEM_API_TARGET: ${{ vars.WORKER1_SYSTEM_API_TARGET }}
         WORKER2_SYSTEM_API_TARGET: ${{ vars.WORKER2_SYSTEM_API_TARGET }}
@@ -101,6 +102,7 @@ jobs:
         build --define image_tag=$IMAGE_TAG
         build --define google_cloud_project=$GCLOUD_PROJECT
         build --define kingdom_system_api_target=$KINGDOM_SYSTEM_API_TARGET
+        build --define kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET
         build --define aggregator_system_api_target=$AGGREGATOR_SYSTEM_API_TARGET
         build --define worker1_system_api_target=$WORKER1_SYSTEM_API_TARGET
         build --define worker2_system_api_target=$WORKER2_SYSTEM_API_TARGET

diff --git a/docs/eks/duchy-deployment.md b/docs/eks/duchy-deployment.md
@@ -217,6 +217,10 @@ files are required in a Duchy:
     -   Set the role (aggregator or non_aggregator) in the config appropriately
     -   [Example](../../src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto)
 
+1.  `worker2_kek.tink`
+    Key encryption key used in HMSS protocol to encrypt tink key pairs.
+    -   [Example](../../src/main/k8s/testing/secretfiles/worker2_kek.tink)
+
 Place these files into the `src/main/k8s/dev/worker2_duchy_secret/` path within
 the Kustomization directory.
 

diff --git a/docs/gke/duchy-deployment.md b/docs/gke/duchy-deployment.md
@@ -213,6 +213,10 @@ files are required in a Duchy:
     -   Set the role (aggregator or non_aggregator) in the config appropriately
     -   [Example](../../src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto)
 
+1.  `worker2_kek.tink`
+    Key encryption key used in HMSS protocol to encrypt tink key pairs.
+    -   [Example](../../src/main/k8s/testing/secretfiles/worker2_kek.tink)
+
 Place these files into the `src/main/k8s/dev/worker1_duchy_secret/` path within
 the Kustomization directory.
 

diff --git a/src/main/docker/images.bzl b/src/main/docker/images.bzl
@@ -26,11 +26,6 @@ COMMON_IMAGES = [
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:async_computation_control_server_image",
         repository = _PREFIX + "/duchy/async-computation-control",
     ),
-    struct(
-        name = "duchy_herald_daemon_image",
-        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:herald_daemon_image",
-        repository = _PREFIX + "/duchy/herald",
-    ),
     struct(
         name = "duchy_spanner_update_schema_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools:update_schema_image",
@@ -106,6 +101,11 @@ COMMON_IMAGES = [
 # List of specs for all Docker containers to push to a container registry.
 # These are only used on GKE.
 GKE_IMAGES = [
+    struct(
+        name = "gcs_herald_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald:gcs_herald_daemon_image",
+        repository = _PREFIX + "/duchy/herald",
+    ),
     struct(
         name = "duchy_computation_control_server_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:gcs_computation_control_server_image",
@@ -131,6 +131,11 @@ GKE_IMAGES = [
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2:gcs_liquid_legions_v2_mill_daemon_image",
         repository = _PREFIX + "/duchy/liquid-legions-v2-mill",
     ),
+    struct(
+        name = "duchy_honest_majority_share_shuffle_mill_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle:gcs_honest_majority_share_shuffle_mill_daemon_image",
+        repository = _PREFIX + "/duchy/honest-majority-share-shuffle-mill",
+    ),
     struct(
         name = "bigquery_edp_simulator_runner_image",
         image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:bigquery_edp_simulator_runner_image",
@@ -146,6 +151,11 @@ GKE_IMAGES = [
 # List of specs for all Docker containers to push to a container registry.
 # These are only used on EKS.
 EKS_IMAGES = [
+    struct(
+        name = "s3_herald_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald:s3_herald_daemon_image",
+        repository = _PREFIX + "/duchy/aws-herald",
+    ),
     struct(
         name = "duchy_s3_computation_control_server_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/server:s3_computation_control_server_image",
@@ -166,6 +176,11 @@ EKS_IMAGES = [
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/liquidlegionsv2:s3_liquid_legions_v2_mill_daemon_image",
         repository = _PREFIX + "/duchy/aws-liquid-legions-v2-mill",
     ),
+    struct(
+        name = "duchy_s3_honest_majority_share_shuffle_mill_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle:s3_honest_majority_share_shuffle_mill_daemon_image",
+        repository = _PREFIX + "/duchy/aws-honest-majority-share-shuffle-mill",
+    ),
     struct(
         name = "duchy_aws_postgres_update_schema_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/postgres/tools:update_schema_image",
@@ -175,11 +190,21 @@ EKS_IMAGES = [
 
 # List of image build rules that are only used locally (e.g. in Kind).
 LOCAL_IMAGES = [
+    struct(
+        name = "forwarded_storage_herald_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:forwarded_storage_herald_daemon_image",
+        repository = _PREFIX + "/duchy/local-herald",
+    ),
     struct(
         name = "forwarded_storage_liquid_legions_v2_mill_daemon_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2:forwarded_storage_liquid_legions_v2_mill_daemon_image",
         repository = _PREFIX + "/duchy/local-liquid-legions-v2-mill",
     ),
+    struct(
+        name = "forwarded_storage_honest_majority_share_shuffle_mill_daemon_image",
+        image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle:forwarded_storage_honest_majority_share_shuffle_mill_daemon_image",
+        repository = _PREFIX + "/duchy/local-honest-majority-share-shuffle-mill",
+    ),
     struct(
         name = "forwarded_storage_computation_control_server_image",
         image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:forwarded_storage_computation_control_server_image",

diff --git a/src/main/k8s/dev/BUILD.bazel b/src/main/k8s/dev/BUILD.bazel
@@ -125,6 +125,7 @@ cue_dump(
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "spanner_instance": GCLOUD_SETTINGS.spanner_instance,
         "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target,
+        "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target,
         "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target,
         "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target,
         "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target,
@@ -143,7 +144,8 @@ cue_dump(
     srcs = ["duchy_gke.cue"],
     cue_tags = {
         "duchy_name": "worker1",
-        "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto",
+        "duchy_protocols_setup_config": "worker1_protocols_setup_config.textproto",
+        "duchy_key_encryption_key_file": "worker1_kek.tink",
         "secret_name": SECRET_NAME,
         "public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name,
         "system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name,
@@ -155,6 +157,7 @@ cue_dump(
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "spanner_instance": GCLOUD_SETTINGS.spanner_instance,
         "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target,
+        "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target,
         "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target,
         "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target,
         "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target,
@@ -173,7 +176,8 @@ cue_dump(
     srcs = ["duchy_gke.cue"],
     cue_tags = {
         "duchy_name": "worker2",
-        "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto",
+        "duchy_protocols_setup_config": "worker2_protocols_setup_config.textproto",
+        "duchy_key_encryption_key_file": "worker2_kek.tink",
         "secret_name": SECRET_NAME,
         "public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name,
         "system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name,
@@ -185,6 +189,7 @@ cue_dump(
         "google_cloud_project": GCLOUD_SETTINGS.project,
         "spanner_instance": GCLOUD_SETTINGS.spanner_instance,
         "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target,
+        "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target,
         "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target,
         "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target,
         "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target,
@@ -203,7 +208,8 @@ cue_dump(
     srcs = ["duchy_eks.cue"],
     cue_tags = {
         "duchy_name": "worker2",
-        "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto",
+        "duchy_protocols_setup_config": "worker2_protocols_setup_config.textproto",
+        "duchy_key_encryption_key_file": "worker2_kek.tink",
         "secret_name": SECRET_NAME,
         "certificate_id": DUCHY_K8S_SETTINGS.certificate_id,
         "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry,
@@ -218,6 +224,7 @@ cue_dump(
         "public_api_eip_allocs": DUCHY_K8S_SETTINGS.public_api_eip_allocs,
         "system_api_eip_allocs": DUCHY_K8S_SETTINGS.system_api_eip_allocs,
         "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target,
+        "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target,
         "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target,
         "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target,
         "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target,
@@ -239,7 +246,7 @@ expand_template(
         "{duchy_id}": "aggregator",
         "{duchy_role}": "aggregator",
     },
-    template = "duchy_secret_kustomization.tmpl.yaml",
+    template = "aggregator_secret_kustomization.tmpl.yaml",
 )
 
 expand_template(
@@ -249,7 +256,7 @@ expand_template(
         "{duchy_id}": "worker1",
         "{duchy_role}": "non_aggregator",
     },
-    template = "duchy_secret_kustomization.tmpl.yaml",
+    template = "non_aggregator_secret_kustomization.tmpl.yaml",
 )
 
 expand_template(
@@ -259,7 +266,7 @@ expand_template(
         "{duchy_id}": "worker2",
         "{duchy_role}": "non_aggregator",
     },
-    template = "duchy_secret_kustomization.tmpl.yaml",
+    template = "non_aggregator_secret_kustomization.tmpl.yaml",
 )
 
 kustomization_dir(

diff --git a/.../dev/duchy_secret_kustomization.tmpl.yaml → ...aggregator_secret_kustomization.tmpl.yaml b/.../dev/duchy_secret_kustomization.tmpl.yaml → ...aggregator_secret_kustomization.tmpl.yaml
diff --git a/src/main/k8s/dev/duchy_eks.cue b/src/main/k8s/dev/duchy_eks.cue
@@ -23,23 +23,30 @@ _systemApiEipAllocs:        string @tag("system_api_eip_allocs")
 _aggregatorSystemApiTarget: string @tag("aggregator_system_api_target")
 _worker1SystemApiTarget:    string @tag("worker1_system_api_target")
 _worker2SystemApiTarget:    string @tag("worker2_system_api_target")
+_duchyKeyEncryptionKeyFile: string @tag("duchy_key_encryption_key_file")
 
 _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)"
 
 #KingdomSystemApiTarget:             string @tag("kingdom_system_api_target")
+#KingdomPublicApiTarget:             string @tag("kingdom_public_api_target")
 #InternalServerServiceAccount:       "internal-server"
 #StorageServiceAccount:              "storage"
 #InternalServerResourceRequirements: #ResourceRequirements & {
 	requests: {
 		cpu: "75m"
 	}
 }
-#HeraldResourceRequirements: #ResourceRequirements & {
+#HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & {
 	requests: {
-		cpu: "25m"
+		cpu:    "25m"
+		memory: "512Mi"
+	}
+	limits: {
+		memory: ResourceRequirements.requests.memory
 	}
 }
-#MillResourceRequirements: ResourceRequirements=#ResourceRequirements & {
+#HeraldMaxHeapSize:            "400M"
+#Llv2MillResourceRequirements: ResourceRequirements=#ResourceRequirements & {
 	requests: {
 		cpu:    "3"
 		memory: "2.5Gi"
@@ -48,9 +55,29 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)"
 		memory: ResourceRequirements.requests.memory
 	}
 }
-#MillMaxHeapSize:        "1G"
-#MillReplicas:           1
-#FulfillmentMaxHeapSize: "96M"
+#Llv2MillMaxHeapSize:          "1G"
+#Llv2MillReplicas:             1
+#HmssMillResourceRequirements: ResourceRequirements=#ResourceRequirements & {
+	requests: {
+		cpu:    "2"
+		memory: "6Gi"
+	}
+	limits: {
+		memory: ResourceRequirements.requests.memory
+	}
+}
+#HmssMillMaxHeapSize:             "5G"
+#HmssMillReplicas:                1
+#FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & {
+	requests: {
+		cpu:    "200m"
+		memory: "512Mi"
+	}
+	limits: {
+		memory: ResourceRequirements.requests.memory
+	}
+}
+#FulfillmentMaxHeapSize: "350M"
 
 objectSets: [
 	default_deny_ingress_and_egress,
@@ -62,16 +89,19 @@ objectSets: [
 
 duchy: #PostgresDuchy & {
 	_imageSuffixes: {
-		"computation-control-server":     "duchy/aws-computation-control"
-		"liquid-legions-v2-mill-daemon":  "duchy/aws-liquid-legions-v2-mill"
-		"requisition-fulfillment-server": "duchy/aws-requisition-fulfillment"
-		"internal-api-server":            "duchy/aws-postgres-internal-server"
-		"update-duchy-schema":            "duchy/aws-postgres-update-schema"
+		"herald-daemon":                             "duchy/aws-herald"
+		"computation-control-server":                "duchy/aws-computation-control"
+		"liquid-legions-v2-mill-daemon":             "duchy/aws-liquid-legions-v2-mill"
+		"honest-majority-share-shuffle-mill-daemon": "duchy/aws-honest-majority-share-shuffle-mill"
+		"requisition-fulfillment-server":            "duchy/aws-requisition-fulfillment"
+		"internal-api-server":                       "duchy/aws-postgres-internal-server"
+		"update-duchy-schema":                       "duchy/aws-postgres-update-schema"
 	}
 	_duchy: {
-		name:                   _duchyName
-		protocols_setup_config: _duchyProtocolsSetupConfig
-		cs_cert_resource_name:  _duchyCertName
+		name:                      _duchyName
+		protocols_setup_config:    _duchyProtocolsSetupConfig
+		cs_cert_resource_name:     _duchyCertName
+		duchyKeyEncryptionKeyFile: _duchyKeyEncryptionKeyFile
 	}
 	_duchy_secret_name: _secretName
 	_computation_control_targets: {
@@ -80,6 +110,7 @@ duchy: #PostgresDuchy & {
 		"worker2":    _worker2SystemApiTarget
 	}
 	_kingdom_system_api_target: #KingdomSystemApiTarget
+	_kingdom_public_api_target: #KingdomPublicApiTarget
 	_blob_storage_flags:        #AwsS3Config.flags
 	_verbose_grpc_logging:      "false"
 	_postgresConfig:            #AwsPostgresConfig
@@ -90,18 +121,34 @@ duchy: #PostgresDuchy & {
 	deployments: {
 		"herald-daemon-deployment": {
 			_container: {
+				_javaOptions: maxHeapSize: #HeraldMaxHeapSize
 				resources: #HeraldResourceRequirements
 			}
-			spec: template: spec: #PodSpec
+			spec: template: spec: #ServiceAccountPodSpec & {
+				serviceAccountName: #StorageServiceAccount
+			}
 		}
 		"liquid-legions-v2-mill-daemon-deployment": {
 			_workLockDuration: "10m"
 			_container: {
-				_javaOptions: maxHeapSize: #MillMaxHeapSize
-				resources: #MillResourceRequirements
+				_javaOptions: maxHeapSize: #Llv2MillMaxHeapSize
+				resources: #Llv2MillResourceRequirements
+			}
+			spec: {
+				replicas: #Llv2MillReplicas
+				template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
+					serviceAccountName: #StorageServiceAccount
+				}
+			}
+		}
+		"honest-majority-share-shuffle-mill-daemon-deployment": {
+			_workLockDuration: "5m"
+			_container: {
+				_javaOptions: maxHeapSize: #HmssMillMaxHeapSize
+				resources: #HmssMillResourceRequirements
 			}
 			spec: {
-				replicas: #MillReplicas
+				replicas: #HmssMillReplicas
 				template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
 					serviceAccountName: #StorageServiceAccount
 				}
@@ -115,6 +162,7 @@ duchy: #PostgresDuchy & {
 		"requisition-fulfillment-server-deployment": {
 			_container: {
 				_javaOptions: maxHeapSize: #FulfillmentMaxHeapSize
+				resources: #FulfillmentResourceRequirements
 			}
 			spec: template: spec: #ServiceAccountPodSpec & {
 				serviceAccountName: #StorageServiceAccount