From d0038d5b2cfb4e9a8162fd8e675f1f2ad798150d Mon Sep 17 00:00:00 2001 From: renjiez Date: Tue, 28 May 2024 20:50:13 +0000 Subject: [PATCH 01/12] Add executable binary classes and docker images for HMSS. --- .github/workflows/configure-aws-duchy.yml | 2 + .github/workflows/configure-duchy.yml | 2 + src/main/docker/images.bzl | 35 ++- src/main/k8s/dev/BUILD.bazel | 19 +- ...aggregator_secret_kustomization.tmpl.yaml} | 0 src/main/k8s/dev/duchy_eks.cue | 67 +++-- src/main/k8s/dev/duchy_gke.cue | 51 +++- .../k8s/dev/kingdom_secret_kustomization.yaml | 1 + ..._aggregator_secret_kustomization.tmpl.yaml | 24 ++ src/main/k8s/duchy.cue | 70 ++++- src/main/k8s/kingdom.cue | 5 + src/main/k8s/local/duchies.cue | 65 +++-- src/main/k8s/testing/secretfiles/BUILD.bazel | 5 + ...ggregator_protocols_setup_config.textproto | 6 + .../hmss_protocol_config_config.textproto | 13 + .../k8s/testing/secretfiles/worker1_kek.tink | 3 + .../worker1_protocols_setup_config.textproto | 16 ++ .../k8s/testing/secretfiles/worker2_kek.tink | 3 + .../worker2_protocols_setup_config.textproto | 16 ++ .../deploy/aws/daemon/herald/BUILD.bazel | 42 +++ .../aws/daemon/herald/S3HeraldDaemon.kt | 38 +++ .../aws/daemon/mill/shareshuffle/BUILD.bazel | 42 +++ .../S3HonestMajorityShareShuffleMillDaemon.kt | 38 +++ .../duchy/deploy/common/ServiceFlags.kt | 23 ++ .../deploy/common/daemon/herald/BUILD.bazel | 29 +- .../herald/ForwardedStorageHeraldDaemon.kt | 35 +++ .../common/daemon/herald/HeraldDaemon.kt | 157 ++++++----- .../daemon/mill/shareshuffle/BUILD.bazel | 66 +++++ ...StorageHonestMajorityShareShuffleDaemon.kt | 37 +++ .../HonestMajorityShareShuffleMillDaemon.kt | 250 ++++++++++++++++++ .../HonestMajorityShareShuffleMillFlags.kt | 150 +++++++++++ .../deploy/gcloud/daemon/herald/BUILD.bazel | 42 +++ .../gcloud/daemon/herald/GcsHeraldDaemon.kt | 38 +++ .../daemon/mill/shareshuffle/BUILD.bazel | 43 +++ ...GcsHonestMajorityShareShuffleMillDaemon.kt | 38 +++ .../integration/common/InProcessDuchy.kt | 2 +- .../deploy/common/HmssProtocolConfig.kt | 5 +- .../kingdom/deploy/common/server/BUILD.bazel | 1 + .../deploy/common/server/KingdomDataServer.kt | 5 + .../common/server/V2alphaPublicApiServer.kt | 32 ++- .../api/v2alpha/MeasurementsService.kt | 16 +- .../service/api/v2alpha/ProtoConversions.kt | 1 + .../internal/kingdom/protocol_config.proto | 7 +- .../kingdom/protocol_config_config.proto | 2 +- .../api/v2alpha/MeasurementsServiceTest.kt | 2 +- 45 files changed, 1381 insertions(+), 163 deletions(-) rename src/main/k8s/dev/{duchy_secret_kustomization.tmpl.yaml => aggregator_secret_kustomization.tmpl.yaml} (100%) create mode 100644 src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml create mode 100644 src/main/k8s/testing/secretfiles/hmss_protocol_config_config.textproto create mode 100644 src/main/k8s/testing/secretfiles/worker1_kek.tink create mode 100644 src/main/k8s/testing/secretfiles/worker1_protocols_setup_config.textproto create mode 100644 src/main/k8s/testing/secretfiles/worker2_kek.tink create mode 100644 src/main/k8s/testing/secretfiles/worker2_protocols_setup_config.textproto create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/BUILD.bazel create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/S3HeraldDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/BUILD.bazel create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/S3HonestMajorityShareShuffleMillDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/ForwardedStorageHeraldDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/ForwardedStorageHonestMajorityShareShuffleDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/BUILD.bazel create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/GcsHeraldDaemon.kt create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/BUILD.bazel create mode 100644 src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/GcsHonestMajorityShareShuffleMillDaemon.kt diff --git a/.github/workflows/configure-aws-duchy.yml b/.github/workflows/configure-aws-duchy.yml index a4d6b941fcf..17f53c892ed 100644 --- a/.github/workflows/configure-aws-duchy.yml +++ b/.github/workflows/configure-aws-duchy.yml @@ -93,6 +93,7 @@ jobs: POSTGRES_HOST: ${{ vars.AWS_POSTGRES_HOST }} POSTGRES_CRED_SECRET_NAME: ${{ vars.AWS_POSTGRES_CRED_SECRET_NAME }} KINGDOM_SYSTEM_API_TARGET: ${{ vars.KINGDOM_SYSTEM_API_TARGET }} + KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }} AGGREGATOR_SYSTEM_API_TARGET: ${{ vars.AGGREGATOR_SYSTEM_API_TARGET }} WORKER1_SYSTEM_API_TARGET: ${{ vars.WORKER1_SYSTEM_API_TARGET }} WORKER2_SYSTEM_API_TARGET: ${{ vars.WORKER2_SYSTEM_API_TARGET }} @@ -105,6 +106,7 @@ jobs: common --config=ghcr build --define image_tag=$IMAGE_TAG build --define kingdom_system_api_target=$KINGDOM_SYSTEM_API_TARGET + build --define kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET build --define aggregator_system_api_target=$AGGREGATOR_SYSTEM_API_TARGET build --define worker1_system_api_target=$WORKER1_SYSTEM_API_TARGET build --define worker2_system_api_target=$WORKER2_SYSTEM_API_TARGET diff --git a/.github/workflows/configure-duchy.yml b/.github/workflows/configure-duchy.yml index 75fc52961b4..77cf42460e1 100644 --- a/.github/workflows/configure-duchy.yml +++ b/.github/workflows/configure-duchy.yml @@ -91,6 +91,7 @@ jobs: env: IMAGE_TAG: ${{ inputs.image-tag }} KINGDOM_SYSTEM_API_TARGET: ${{ vars.KINGDOM_SYSTEM_API_TARGET }} + KINGDOM_PUBLIC_API_TARGET: ${{ vars.KINGDOM_PUBLIC_API_TARGET }} AGGREGATOR_SYSTEM_API_TARGET: ${{ vars.AGGREGATOR_SYSTEM_API_TARGET }} WORKER1_SYSTEM_API_TARGET: ${{ vars.WORKER1_SYSTEM_API_TARGET }} WORKER2_SYSTEM_API_TARGET: ${{ vars.WORKER2_SYSTEM_API_TARGET }} @@ -101,6 +102,7 @@ jobs: build --define image_tag=$IMAGE_TAG build --define google_cloud_project=$GCLOUD_PROJECT build --define kingdom_system_api_target=$KINGDOM_SYSTEM_API_TARGET + build --define kingdom_public_api_target=$KINGDOM_PUBLIC_API_TARGET build --define aggregator_system_api_target=$AGGREGATOR_SYSTEM_API_TARGET build --define worker1_system_api_target=$WORKER1_SYSTEM_API_TARGET build --define worker2_system_api_target=$WORKER2_SYSTEM_API_TARGET diff --git a/src/main/docker/images.bzl b/src/main/docker/images.bzl index bf55fcbb61a..1d3ee0553cb 100644 --- a/src/main/docker/images.bzl +++ b/src/main/docker/images.bzl @@ -26,11 +26,6 @@ COMMON_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:async_computation_control_server_image", repository = _PREFIX + "/duchy/async-computation-control", ), - struct( - name = "duchy_herald_daemon_image", - image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:herald_daemon_image", - repository = _PREFIX + "/duchy/herald", - ), struct( name = "duchy_spanner_update_schema_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/tools:update_schema_image", @@ -106,6 +101,11 @@ COMMON_IMAGES = [ # List of specs for all Docker containers to push to a container registry. # These are only used on GKE. GKE_IMAGES = [ + struct( + name = "gcs_herald_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald:gcs_herald_daemon_image", + repository = _PREFIX + "/duchy/herald", + ), struct( name = "duchy_computation_control_server_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/server:gcs_computation_control_server_image", @@ -131,6 +131,11 @@ GKE_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/liquidlegionsv2:gcs_liquid_legions_v2_mill_daemon_image", repository = _PREFIX + "/duchy/liquid-legions-v2-mill", ), + struct( + name = "duchy_honest_majority_share_shuffle_mill_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle:gcs_honest_majority_share_shuffle_mill_daemon_image", + repository = _PREFIX + "/duchy/honest-majority-share-shuffle-mill", + ), struct( name = "bigquery_edp_simulator_runner_image", image = "//src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider:bigquery_edp_simulator_runner_image", @@ -146,6 +151,11 @@ GKE_IMAGES = [ # List of specs for all Docker containers to push to a container registry. # These are only used on EKS. EKS_IMAGES = [ + struct( + name = "s3_herald_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald:s3_herald_daemon_image", + repository = _PREFIX + "/duchy/aws-herald", + ), struct( name = "duchy_s3_computation_control_server_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/server:s3_computation_control_server_image", @@ -166,6 +176,11 @@ EKS_IMAGES = [ image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/liquidlegionsv2:s3_liquid_legions_v2_mill_daemon_image", repository = _PREFIX + "/duchy/aws-liquid-legions-v2-mill", ), + struct( + name = "duchy_s3_honest_majority_share_shuffle_mill_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle:s3_honest_majority_share_shuffle_mill_daemon_image", + repository = _PREFIX + "/duchy/aws-honest-majority-share-shuffle-mill", + ), struct( name = "duchy_aws_postgres_update_schema_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/postgres/tools:update_schema_image", @@ -175,11 +190,21 @@ EKS_IMAGES = [ # List of image build rules that are only used locally (e.g. in Kind). LOCAL_IMAGES = [ + struct( + name = "forwarded_storage_herald_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:forwarded_storage_herald_daemon_image", + repository = _PREFIX + "/duchy/local-herald", + ), struct( name = "forwarded_storage_liquid_legions_v2_mill_daemon_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/liquidlegionsv2:forwarded_storage_liquid_legions_v2_mill_daemon_image", repository = _PREFIX + "/duchy/local-liquid-legions-v2-mill", ), + struct( + name = "forwarded_storage_honest_majority_share_shuffle_mill_daemon_image", + image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle:forwarded_storage_honest_majority_share_shuffle_mill_daemon_image", + repository = _PREFIX + "/duchy/local-honest-majority-share-shuffle-mill", + ), struct( name = "forwarded_storage_computation_control_server_image", image = "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/server:forwarded_storage_computation_control_server_image", diff --git a/src/main/k8s/dev/BUILD.bazel b/src/main/k8s/dev/BUILD.bazel index 6211faac1c7..5657f506fc9 100644 --- a/src/main/k8s/dev/BUILD.bazel +++ b/src/main/k8s/dev/BUILD.bazel @@ -125,6 +125,7 @@ cue_dump( "google_cloud_project": GCLOUD_SETTINGS.project, "spanner_instance": GCLOUD_SETTINGS.spanner_instance, "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target, "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target, "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target, @@ -143,7 +144,8 @@ cue_dump( srcs = ["duchy_gke.cue"], cue_tags = { "duchy_name": "worker1", - "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto", + "duchy_protocols_setup_config": "worker1_protocols_setup_config.textproto", + "duchy_key_encryption_key_file": "worker1_kek.tink", "secret_name": SECRET_NAME, "public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name, "system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name, @@ -155,6 +157,7 @@ cue_dump( "google_cloud_project": GCLOUD_SETTINGS.project, "spanner_instance": GCLOUD_SETTINGS.spanner_instance, "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target, "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target, "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target, @@ -173,7 +176,8 @@ cue_dump( srcs = ["duchy_gke.cue"], cue_tags = { "duchy_name": "worker2", - "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto", + "duchy_protocols_setup_config": "worker2_protocols_setup_config.textproto", + "duchy_key_encryption_key_file": "worker2_kek.tink", "secret_name": SECRET_NAME, "public_api_address_name": DUCHY_K8S_SETTINGS.public_api_address_name, "system_api_address_name": DUCHY_K8S_SETTINGS.system_api_address_name, @@ -185,6 +189,7 @@ cue_dump( "google_cloud_project": GCLOUD_SETTINGS.project, "spanner_instance": GCLOUD_SETTINGS.spanner_instance, "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target, "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target, "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target, @@ -203,7 +208,8 @@ cue_dump( srcs = ["duchy_eks.cue"], cue_tags = { "duchy_name": "worker2", - "duchy_protocols_setup_config": "non_aggregator_protocols_setup_config.textproto", + "duchy_protocols_setup_config": "worker2_protocols_setup_config.textproto", + "duchy_key_encryption_key_file": "worker2_kek.tink", "secret_name": SECRET_NAME, "certificate_id": DUCHY_K8S_SETTINGS.certificate_id, "container_registry": IMAGE_REPOSITORY_SETTINGS.container_registry, @@ -218,6 +224,7 @@ cue_dump( "public_api_eip_allocs": DUCHY_K8S_SETTINGS.public_api_eip_allocs, "system_api_eip_allocs": DUCHY_K8S_SETTINGS.system_api_eip_allocs, "kingdom_system_api_target": KINGDOM_K8S_SETTINGS.system_api_target, + "kingdom_public_api_target": KINGDOM_K8S_SETTINGS.public_api_target, "aggregator_system_api_target": DUCHY_K8S_SETTINGS.aggregator_system_api_target, "worker1_system_api_target": DUCHY_K8S_SETTINGS.worker1_system_api_target, "worker2_system_api_target": DUCHY_K8S_SETTINGS.worker2_system_api_target, @@ -239,7 +246,7 @@ expand_template( "{duchy_id}": "aggregator", "{duchy_role}": "aggregator", }, - template = "duchy_secret_kustomization.tmpl.yaml", + template = "aggregator_secret_kustomization.tmpl.yaml", ) expand_template( @@ -249,7 +256,7 @@ expand_template( "{duchy_id}": "worker1", "{duchy_role}": "non_aggregator", }, - template = "duchy_secret_kustomization.tmpl.yaml", + template = "non_aggregator_secret_kustomization.tmpl.yaml", ) expand_template( @@ -259,7 +266,7 @@ expand_template( "{duchy_id}": "worker2", "{duchy_role}": "non_aggregator", }, - template = "duchy_secret_kustomization.tmpl.yaml", + template = "non_aggregator_secret_kustomization.tmpl.yaml", ) kustomization_dir( diff --git a/src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml b/src/main/k8s/dev/aggregator_secret_kustomization.tmpl.yaml similarity index 100% rename from src/main/k8s/dev/duchy_secret_kustomization.tmpl.yaml rename to src/main/k8s/dev/aggregator_secret_kustomization.tmpl.yaml diff --git a/src/main/k8s/dev/duchy_eks.cue b/src/main/k8s/dev/duchy_eks.cue index cc8ddb3b58d..f2a941d27de 100644 --- a/src/main/k8s/dev/duchy_eks.cue +++ b/src/main/k8s/dev/duchy_eks.cue @@ -23,10 +23,12 @@ _systemApiEipAllocs: string @tag("system_api_eip_allocs") _aggregatorSystemApiTarget: string @tag("aggregator_system_api_target") _worker1SystemApiTarget: string @tag("worker1_system_api_target") _worker2SystemApiTarget: string @tag("worker2_system_api_target") +_duchyKeyEncryptionKeyFile: string @tag("duchy_key_encryption_key_file") _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)" #KingdomSystemApiTarget: string @tag("kingdom_system_api_target") +#KingdomPublicApiTarget: string @tag("kingdom_public_api_target") #InternalServerServiceAccount: "internal-server" #StorageServiceAccount: "storage" #InternalServerResourceRequirements: #ResourceRequirements & { @@ -34,23 +36,37 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)" cpu: "75m" } } -#HeraldResourceRequirements: #ResourceRequirements & { +#HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { - cpu: "25m" + cpu: "25m" + memory: "1024Mi" + } + limits: { + memory: ResourceRequirements.requests.memory } } +#HeraldMaxHeapSize: "800M" #MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "3" - memory: "2.5Gi" + memory: "4Gi" + } + limits: { + memory: ResourceRequirements.requests.memory + } +} +#MillMaxHeapSize: "3500M" +#MillReplicas: 1 +#FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & { + requests: { + cpu: "200m" + memory: "512Mi" } limits: { memory: ResourceRequirements.requests.memory } } -#MillMaxHeapSize: "1G" -#MillReplicas: 1 -#FulfillmentMaxHeapSize: "96M" +#FulfillmentMaxHeapSize: "350M" objectSets: [ default_deny_ingress_and_egress, @@ -62,16 +78,19 @@ objectSets: [ duchy: #PostgresDuchy & { _imageSuffixes: { - "computation-control-server": "duchy/aws-computation-control" - "liquid-legions-v2-mill-daemon": "duchy/aws-liquid-legions-v2-mill" - "requisition-fulfillment-server": "duchy/aws-requisition-fulfillment" - "internal-api-server": "duchy/aws-postgres-internal-server" - "update-duchy-schema": "duchy/aws-postgres-update-schema" + "herald-daemon": "duchy/aws-herald" + "computation-control-server": "duchy/aws-computation-control" + "liquid-legions-v2-mill-daemon": "duchy/aws-liquid-legions-v2-mill" + "honest-majority-share-shuffle-mill-daemon": "duchy/aws-honest-majority-share-shuffle-mill" + "requisition-fulfillment-server": "duchy/aws-requisition-fulfillment" + "internal-api-server": "duchy/aws-postgres-internal-server" + "update-duchy-schema": "duchy/aws-postgres-update-schema" } _duchy: { - name: _duchyName - protocols_setup_config: _duchyProtocolsSetupConfig - cs_cert_resource_name: _duchyCertName + name: _duchyName + protocols_setup_config: _duchyProtocolsSetupConfig + cs_cert_resource_name: _duchyCertName + duchyKeyEncryptionKeyFile: _duchyKeyEncryptionKeyFile } _duchy_secret_name: _secretName _computation_control_targets: { @@ -80,6 +99,7 @@ duchy: #PostgresDuchy & { "worker2": _worker2SystemApiTarget } _kingdom_system_api_target: #KingdomSystemApiTarget + _kingdom_public_api_target: #KingdomPublicApiTarget _blob_storage_flags: #AwsS3Config.flags _verbose_grpc_logging: "false" _postgresConfig: #AwsPostgresConfig @@ -90,9 +110,12 @@ duchy: #PostgresDuchy & { deployments: { "herald-daemon-deployment": { _container: { + _javaOptions: maxHeapSize: #HeraldMaxHeapSize resources: #HeraldResourceRequirements } - spec: template: spec: #PodSpec + spec: template: spec: #ServiceAccountPodSpec & { + serviceAccountName: #StorageServiceAccount + } } "liquid-legions-v2-mill-daemon-deployment": { _workLockDuration: "10m" @@ -107,6 +130,19 @@ duchy: #PostgresDuchy & { } } } + "honest-majority-share-shuffle-mill-daemon-deployment": { + _workLockDuration: "5m" + _container: { + _javaOptions: maxHeapSize: #MillMaxHeapSize + resources: #MillResourceRequirements + } + spec: { + replicas: #MillReplicas + template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { + serviceAccountName: #StorageServiceAccount + } + } + } "computation-control-server-deployment": { spec: template: spec: #ServiceAccountPodSpec & { serviceAccountName: #StorageServiceAccount @@ -115,6 +151,7 @@ duchy: #PostgresDuchy & { "requisition-fulfillment-server-deployment": { _container: { _javaOptions: maxHeapSize: #FulfillmentMaxHeapSize + resources: #FulfillmentResourceRequirements } spec: template: spec: #ServiceAccountPodSpec & { serviceAccountName: #StorageServiceAccount diff --git a/src/main/k8s/dev/duchy_gke.cue b/src/main/k8s/dev/duchy_gke.cue index b6bd340ce9d..96ea64204ea 100644 --- a/src/main/k8s/dev/duchy_gke.cue +++ b/src/main/k8s/dev/duchy_gke.cue @@ -24,10 +24,12 @@ _systemApiAddressName: string @tag("system_api_address_name") _aggregatorSystemApiTarget: string @tag("aggregator_system_api_target") _worker1SystemApiTarget: string @tag("worker1_system_api_target") _worker2SystemApiTarget: string @tag("worker2_system_api_target") +_duchyKeyEncryptionKeyFile: string @tag("duchy_key_encryption_key_file") _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)" #KingdomSystemApiTarget: string @tag("kingdom_system_api_target") +#KingdomPublicApiTarget: string @tag("kingdom_public_api_target") #InternalServerServiceAccount: "internal-server" #StorageServiceAccount: "storage" #InternalServerResourceRequirements: #ResourceRequirements & { @@ -35,23 +37,37 @@ _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)" cpu: "75m" } } -#HeraldResourceRequirements: #ResourceRequirements & { +#HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { - cpu: "25m" + cpu: "25m" + memory: "512M" + } + limits: { + memory: ResourceRequirements.requests.memory } } +#HeraldMaxHeapSize: "400M" #MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "3" - memory: "2.5Gi" + memory: "6Gi" + } + limits: { + memory: ResourceRequirements.requests.memory + } +} +#MillMaxHeapSize: "5G" +#MillReplicas: 1 +#FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & { + requests: { + cpu: "200m" + memory: "512Mi" } limits: { memory: ResourceRequirements.requests.memory } } -#MillMaxHeapSize: "1G" -#MillReplicas: 1 -#FulfillmentMaxHeapSize: "96M" +#FulfillmentMaxHeapSize: "350M" objectSets: [ default_deny_ingress_and_egress, @@ -72,6 +88,9 @@ duchy: #SpannerDuchy & { name: _duchy_name protocols_setup_config: _duchy_protocols_setup_config cs_cert_resource_name: _duchy_cert_name + if (_duchyKeyEncryptionKeyFile != _|_) { + duchyKeyEncryptionKeyFile: _duchyKeyEncryptionKeyFile + } } _duchy_secret_name: _secret_name _computation_control_targets: { @@ -80,6 +99,7 @@ duchy: #SpannerDuchy & { "worker2": _worker2SystemApiTarget } _kingdom_system_api_target: #KingdomSystemApiTarget + _kingdom_public_api_target: #KingdomPublicApiTarget _blob_storage_flags: _cloudStorageConfig.flags _verbose_grpc_logging: "false" _duchyMillParallelism: 4 @@ -107,9 +127,12 @@ duchy: #SpannerDuchy & { } "herald-daemon-deployment": { _container: { + _javaOptions: maxHeapSize: #HeraldMaxHeapSize resources: #HeraldResourceRequirements } - spec: template: spec: #SpotVmPodSpec + spec: template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { + serviceAccountName: #StorageServiceAccount + } } "liquid-legions-v2-mill-daemon-deployment": { _workLockDuration: "10m" @@ -124,6 +147,19 @@ duchy: #SpannerDuchy & { } } } + "honest-majority-share-shuffle-mill-daemon-deployment": { + _workLockDuration: "5m" + _container: { + _javaOptions: maxHeapSize: #MillMaxHeapSize + resources: #MillResourceRequirements + } + spec: { + replicas: #MillReplicas + template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { + serviceAccountName: #StorageServiceAccount + } + } + } "computation-control-server-deployment": { spec: template: spec: #ServiceAccountPodSpec & { serviceAccountName: #StorageServiceAccount @@ -132,6 +168,7 @@ duchy: #SpannerDuchy & { "requisition-fulfillment-server-deployment": { _container: { _javaOptions: maxHeapSize: #FulfillmentMaxHeapSize + resources: #FulfillmentResourceRequirements } spec: template: spec: #ServiceAccountPodSpec & { serviceAccountName: #StorageServiceAccount diff --git a/src/main/k8s/dev/kingdom_secret_kustomization.yaml b/src/main/k8s/dev/kingdom_secret_kustomization.yaml index acb8590e1aa..d916807a466 100644 --- a/src/main/k8s/dev/kingdom_secret_kustomization.yaml +++ b/src/main/k8s/dev/kingdom_secret_kustomization.yaml @@ -23,3 +23,4 @@ secretGenerator: - duchy_id_config.textproto - llv2_protocol_config_config.textproto - ro_llv2_protocol_config_config.textproto + - hmss_protocol_config_config.textproto diff --git a/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml b/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml new file mode 100644 index 00000000000..d1e33e75e29 --- /dev/null +++ b/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml @@ -0,0 +1,24 @@ +# Copyright 2024 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +secretGenerator: +- name: certs-and-configs + files: + - all_root_certs.pem + - {duchy_id}_tls.pem + - {duchy_id}_tls.key + - {duchy_id}_cs_cert.der + - {duchy_id}_cs_private.der + - duchy_cert_config.textproto + - {duchy_id}_protocols_setup_config.textproto + - {duchy_id}_kek.tink \ No newline at end of file diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index 05228823919..5807393850e 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -20,9 +20,10 @@ import ("strings") #Duchy: { _duchy: { - name: string - protocols_setup_config: string - cs_cert_resource_name: string + name: string + protocols_setup_config: string + cs_cert_resource_name: string + duchyKeyEncryptionKeyFile: string } _duchy_secret_name: string _computation_control_targets: [Name=_]: string @@ -30,23 +31,26 @@ import ("strings") _computationsTimeToLive: string | *"180d" _duchyMillParallelism: uint | *2 _kingdom_system_api_target: string + _kingdom_public_api_target: string _blob_storage_flags: [...string] _verbose_grpc_logging: "true" | "false" - _name: _duchy.name - _protocols_setup_config: _duchy.protocols_setup_config - _cs_cert_resource_name: _duchy.cs_cert_resource_name + _name: _duchy.name + _protocols_setup_config: _duchy.protocols_setup_config + _cs_cert_resource_name: _duchy.cs_cert_resource_name + _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile _object_prefix: "\(_name)-" _imageSuffixes: [string]: string _imageSuffixes: { - "async-computation-control-server": string | *"duchy/async-computation-control" - "computation-control-server": string | *"duchy/computation-control" - "herald-daemon": string | *"duchy/herald" - "liquid-legions-v2-mill-daemon": string | *"duchy/liquid-legions-v2-mill" - "requisition-fulfillment-server": string | *"duchy/requisition-fulfillment" - "computations-cleaner": string | *"duchy/computations-cleaner" + "async-computation-control-server": string | *"duchy/async-computation-control" + "computation-control-server": string | *"duchy/computation-control" + "herald-daemon": string | *"duchy/herald" + "liquid-legions-v2-mill-daemon": string | *"duchy/liquid-legions-v2-mill" + "honest-majority-share-shuffle-mill-daemon": string | *"duchy/honest-majority-share-shuffle-mill" + "requisition-fulfillment-server": string | *"duchy/requisition-fulfillment" + "computations-cleaner": string | *"duchy/computations-cleaner" } _imageConfigs: [string]: #ImageConfig _imageConfigs: { @@ -72,6 +76,7 @@ import ("strings") _duchy_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/\(_name)_tls.pem" _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" + _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" @@ -83,6 +88,8 @@ import ("strings") _duchyDeletableStatesFlag: [ for state in _deletableComputationStates {"--deletable-computation-state=\(state)"}] _kingdom_system_api_target_flag: "--kingdom-system-api-target=\(_kingdom_system_api_target)" _kingdom_system_api_cert_host_flag: "--kingdom-system-api-cert-host=localhost" + _kingdom_public_api_target_flag: "--kingdom-public-api-target=\(_kingdom_public_api_target)" + _kingdom_public_api_cert_host_flag: "--kingdom-public-api-cert-host=localhost" _debug_verbose_grpc_client_logging_flag: "--debug-verbose-grpc-client-logging=\(_verbose_grpc_logging)" _debug_verbose_grpc_server_logging_flag: "--debug-verbose-grpc-server-logging=\(_verbose_grpc_logging)" _computation_control_target_flags: [ for duchyId, target in _computation_control_targets {"--duchy-computation-control-target=\(duchyId)=\(target)"}] @@ -123,8 +130,9 @@ import ("strings") _duchy_protocols_setup_config_flag, _kingdom_system_api_target_flag, _kingdom_system_api_cert_host_flag, + if (_duchyKeyEncryptionKeyFile != _|_) {_duchyKeyEncryptionKeyFileFlag}, _debug_verbose_grpc_client_logging_flag, - ] + _duchyDeletableStatesFlag + ] + _blob_storage_flags + _duchyDeletableStatesFlag spec: template: spec: _dependencies: [ "\(_name)-internal-api-server", ] @@ -154,6 +162,34 @@ import ("strings") "\(_name)-internal-api-server", "\(_name)-computation-control-server", ] } + "honest-majority-share-shuffle-mill-daemon-deployment": Deployment={ + _workLockDuration?: string + _container: args: [ + _duchyInternalApiTargetFlag, + _duchyInternalApiCertHostFlag, + _duchy_name_flag, + _duchy_info_config_flag, + _duchy_tls_cert_file_flag, + _duchy_tls_key_file_flag, + _duchy_cert_collection_file_flag, + _duchy_cs_cert_file_flag, + _duchy_cs_key_file_flag, + _duchy_cs_cert_rename_name_flag, + _duchy_protocols_setup_config_flag, + _kingdom_system_api_target_flag, + _kingdom_system_api_cert_host_flag, + _kingdom_public_api_target_flag, + _kingdom_public_api_cert_host_flag, + if (_duchyKeyEncryptionKeyFile != _|_) {_duchyKeyEncryptionKeyFileFlag}, + if (_millPollingInterval != _|_) {"--polling-interval=\(_millPollingInterval)"}, + if (_workLockDuration != _|_) {"--work-lock-duration=\(_workLockDuration)"}, + _otlpEndpoint, + "--otel-service-name=\(Deployment.metadata.name)", + ] + _blob_storage_flags + _computation_control_target_flags + spec: template: spec: _dependencies: [ + "\(_name)-internal-api-server", "\(_name)-computation-control-server", + ] + } "async-computation-control-server-deployment": #ServerDeployment & { _container: args: [ _duchyInternalApiTargetFlag, @@ -265,6 +301,7 @@ import ("strings") _sourceMatchLabels: [ _object_prefix + "herald-daemon-app", _object_prefix + "liquid-legions-v2-mill-daemon-app", + _object_prefix + "honest-majority-share-shuffle-mill-daemon-app", _object_prefix + "async-computation-control-server-app", _object_prefix + "requisition-fulfillment-server-app", _object_prefix + "computations-cleaner-app", @@ -321,6 +358,13 @@ import ("strings") any: {} } } + "honest-majority-share-shuffle-mill-daemon": { + _app_label: _object_prefix + "honest-majority-share-shuffle-mill-daemon-app" + _egresses: { + // Need to send external traffic. + any: {} + } + } "herald-daemon": { _app_label: _object_prefix + "herald-daemon-app" _egresses: { diff --git a/src/main/k8s/kingdom.cue b/src/main/k8s/kingdom.cue index 707bc6f23c4..af9b28b0188 100644 --- a/src/main/k8s/kingdom.cue +++ b/src/main/k8s/kingdom.cue @@ -60,6 +60,9 @@ import ("strings") _llv2_protocol_config_config: "--llv2-protocol-config-config=/var/run/secrets/files/llv2_protocol_config_config.textproto" _ro_llv2_protocol_config_config: "--ro-llv2-protocol-config-config=/var/run/secrets/files/ro_llv2_protocol_config_config.textproto" _ro_llv2_enable_flag: "--enable-ro-llv2-protocol" + _hmssRfEnableFlag: "--enable-hmss-for-rf" + _hmssReachEnableFlag: "--enable-hmss-for-reach" + _hmssProtocolConfigConfig: "--hmss-protocol-config-config=/var/run/secrets/files/hmss_protocol_config_config.textproto" _kingdom_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/kingdom_tls.pem" _kingdom_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/kingdom_tls.key" _kingdom_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" @@ -130,6 +133,7 @@ import ("strings") _duchy_id_config_flag, _llv2_protocol_config_config, _ro_llv2_protocol_config_config, + _hmssProtocolConfigConfig, _kingdom_tls_cert_file_flag, _kingdom_tls_key_file_flag, // Internal Kingdom API server should only trust Kingdom certs. @@ -174,6 +178,7 @@ import ("strings") _debug_verbose_grpc_server_logging_flag, _llv2_protocol_config_config, _ro_llv2_protocol_config_config, + _hmssProtocolConfigConfig, _ro_llv2_enable_flag, _kingdom_tls_cert_file_flag, _kingdom_tls_key_file_flag, diff --git a/src/main/k8s/local/duchies.cue b/src/main/k8s/local/duchies.cue index e3d312363a8..b8eec545034 100644 --- a/src/main/k8s/local/duchies.cue +++ b/src/main/k8s/local/duchies.cue @@ -21,7 +21,21 @@ _worker1_cert_name: string @tag("worker1_cert_name") _worker2_cert_name: string @tag("worker2_cert_name") #KingdomSystemApiTarget: (#Target & {name: "system-api-server"}).target +#KingdomPublicApiTarget: (#Target & {name: "v2alpha-public-api-server"}).target #SpannerEmulatorHost: (#Target & {name: "spanner-emulator"}).target + +#MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { + requests: { + cpu: "3" + memory: "4Gi" + } + limits: { + memory: ResourceRequirements.requests.memory + } +} +#MillMaxHeapSize: "3500M" +#MillReplicas: 1 + #DuchyConfig: { let duchyName = name name: string @@ -40,14 +54,16 @@ _duchyConfigs: { databaseType: "spanner" } "worker1": { - protocolsSetupConfig: "non_aggregator_protocols_setup_config.textproto" - certificateResourceName: _worker1_cert_name - databaseType: "spanner" + protocolsSetupConfig: "worker1_protocols_setup_config.textproto" + certificateResourceName: _worker1_cert_name + databaseType: "spanner" + duchyKeyEncryptionKeyFile: "worker1_kek.tink" } "worker2": { - protocolsSetupConfig: "non_aggregator_protocols_setup_config.textproto" - certificateResourceName: _worker2_cert_name - databaseType: "postgres" + protocolsSetupConfig: "worker2_protocols_setup_config.textproto" + certificateResourceName: _worker2_cert_name + databaseType: "postgres" + duchyKeyEncryptionKeyFile: "worker2_kek.tink" } } @@ -78,13 +94,16 @@ _computationControlTargets: { _baseDuchyConfig: { _imageSuffixes: { - "computation-control-server": "duchy/local-computation-control" - "liquid-legions-v2-mill-daemon": "duchy/local-liquid-legions-v2-mill" - "requisition-fulfillment-server": "duchy/local-requisition-fulfillment" + "computation-control-server": "duchy/local-computation-control" + "herald-daemon": "duchy/local-herald" + "liquid-legions-v2-mill-daemon": "duchy/local-liquid-legions-v2-mill" + "honest-majority-share-shuffle-mill-daemon": "duchy/local-honest-majority-share-shuffle-mill" + "requisition-fulfillment-server": "duchy/local-requisition-fulfillment" } _duchy_secret_name: _secret_name _computation_control_targets: _computationControlTargets _kingdom_system_api_target: #KingdomSystemApiTarget + _kingdom_public_api_target: #KingdomPublicApiTarget _blob_storage_flags: [ "--forwarded-storage-service-target=" + (#Target & {name: "fake-storage-server"}).target, "--forwarded-storage-cert-host=localhost", @@ -100,9 +119,18 @@ duchies: [ "internal-api-server": "duchy/local-spanner-computations" } _duchy: { - name: duchyConfig.name - protocols_setup_config: duchyConfig.protocolsSetupConfig - cs_cert_resource_name: duchyConfig.certificateResourceName + name: duchyConfig.name + protocols_setup_config: duchyConfig.protocolsSetupConfig + cs_cert_resource_name: duchyConfig.certificateResourceName + duchyKeyEncryptionKeyFile: duchyConfig.duchyKeyEncryptionKeyFile + } + deployments: { + "honest-majority-share-shuffle-mill-daemon-deployment": { + _container: { + _javaOptions: maxHeapSize: #MillMaxHeapSize + resources: #MillResourceRequirements + } + } } } } @@ -112,9 +140,10 @@ duchies: [ "internal-api-server": "duchy/local-postgres-internal-server" } _duchy: { - name: duchyConfig.name - protocols_setup_config: duchyConfig.protocolsSetupConfig - cs_cert_resource_name: duchyConfig.certificateResourceName + name: duchyConfig.name + protocols_setup_config: duchyConfig.protocolsSetupConfig + cs_cert_resource_name: duchyConfig.certificateResourceName + duchyKeyEncryptionKeyFile: duchyConfig.duchyKeyEncryptionKeyFile } _postgresConfig: { serviceName: "postgres" @@ -126,6 +155,12 @@ duchies: [ _container: _envVars: EnvVars _updateSchemaContainer: _envVars: EnvVars } + "honest-majority-share-shuffle-mill-daemon-deployment": { + _container: { + _javaOptions: maxHeapSize: #MillMaxHeapSize + resources: #MillResourceRequirements + } + } } } } diff --git a/src/main/k8s/testing/secretfiles/BUILD.bazel b/src/main/k8s/testing/secretfiles/BUILD.bazel index 9bdbfe3f2a1..971f460fd03 100644 --- a/src/main/k8s/testing/secretfiles/BUILD.bazel +++ b/src/main/k8s/testing/secretfiles/BUILD.bazel @@ -147,10 +147,14 @@ SECRET_FILES = [ "worker1_tls.pem", "worker1_cs_cert.der", "worker1_cs_private.der", + "worker1_kek.tink", + "worker1_protocols_setup_config.textproto", "worker2_tls.key", "worker2_tls.pem", "worker2_cs_cert.der", "worker2_cs_private.der", + "worker2_kek.tink", + "worker2_protocols_setup_config.textproto", "edp1_cs_cert.der", "edp1_cs_private.der", "edp1_enc_private.tink", @@ -207,6 +211,7 @@ SECRET_FILES = [ "non_aggregator_protocols_setup_config.textproto", "llv2_protocol_config_config.textproto", "ro_llv2_protocol_config_config.textproto", + "hmss_protocol_config_config.textproto", "exchange_workflow.textproto", "reporting_tls.key", "reporting_tls.pem", diff --git a/src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto b/src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto index 3e78df01981..0844501a583 100644 --- a/src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto +++ b/src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto @@ -8,3 +8,9 @@ reach_only_liquid_legions_v2 { role: AGGREGATOR external_aggregator_duchy_id: "aggregator" } +honest_majority_share_shuffle { + role: AGGREGATOR + first_non_aggregator_duchy_id: "worker1" + second_non_aggregator_duchy_id: "worker2" + aggregator_duchy_id: "aggregator" +} diff --git a/src/main/k8s/testing/secretfiles/hmss_protocol_config_config.textproto b/src/main/k8s/testing/secretfiles/hmss_protocol_config_config.textproto new file mode 100644 index 00000000000..2a28edf7945 --- /dev/null +++ b/src/main/k8s/testing/secretfiles/hmss_protocol_config_config.textproto @@ -0,0 +1,13 @@ +# proto-file: src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto +# proto-message: HmssProtocolConfigConfig +protocol_config { + sketch_params { + bytes_per_register: 2 + ring_modulus: 127 + } + noise_mechanism: DISCRETE_GAUSSIAN +} + +required_external_duchy_ids: "worker1" +required_external_duchy_ids: "worker2" +required_external_duchy_ids: "aggregator" \ No newline at end of file diff --git a/src/main/k8s/testing/secretfiles/worker1_kek.tink b/src/main/k8s/testing/secretfiles/worker1_kek.tink new file mode 100644 index 00000000000..ec7a9c62aec --- /dev/null +++ b/src/main/k8s/testing/secretfiles/worker1_kek.tink @@ -0,0 +1,3 @@ +T +H +0type.googleapis.com/google.crypto.tink.AesGcmKey pp}k5  \ No newline at end of file diff --git a/src/main/k8s/testing/secretfiles/worker1_protocols_setup_config.textproto b/src/main/k8s/testing/secretfiles/worker1_protocols_setup_config.textproto new file mode 100644 index 00000000000..9c494f34f6f --- /dev/null +++ b/src/main/k8s/testing/secretfiles/worker1_protocols_setup_config.textproto @@ -0,0 +1,16 @@ +# proto-file: src/main/proto/wfa/measurement/internal/duchy/config/protocols_setup_config.proto +# proto-message: ProtocolsSetupConfig +liquid_legions_v2 { + role: NON_AGGREGATOR + external_aggregator_duchy_id: "aggregator" +} +reach_only_liquid_legions_v2 { + role: NON_AGGREGATOR + external_aggregator_duchy_id: "aggregator" +} +honest_majority_share_shuffle { + role: FIRST_NON_AGGREGATOR + first_non_aggregator_duchy_id: "worker1" + second_non_aggregator_duchy_id: "worker2" + aggregator_duchy_id: "aggregator" +} \ No newline at end of file diff --git a/src/main/k8s/testing/secretfiles/worker2_kek.tink b/src/main/k8s/testing/secretfiles/worker2_kek.tink new file mode 100644 index 00000000000..40b183a9aaa --- /dev/null +++ b/src/main/k8s/testing/secretfiles/worker2_kek.tink @@ -0,0 +1,3 @@ +'S +H +0type.googleapis.com/google.crypto.tink.AesGcmKeyCK(1/?'  \ No newline at end of file diff --git a/src/main/k8s/testing/secretfiles/worker2_protocols_setup_config.textproto b/src/main/k8s/testing/secretfiles/worker2_protocols_setup_config.textproto new file mode 100644 index 00000000000..871517c93c9 --- /dev/null +++ b/src/main/k8s/testing/secretfiles/worker2_protocols_setup_config.textproto @@ -0,0 +1,16 @@ +# proto-file: src/main/proto/wfa/measurement/internal/duchy/config/protocols_setup_config.proto +# proto-message: ProtocolsSetupConfig +liquid_legions_v2 { + role: NON_AGGREGATOR + external_aggregator_duchy_id: "aggregator" +} +reach_only_liquid_legions_v2 { + role: NON_AGGREGATOR + external_aggregator_duchy_id: "aggregator" +} +honest_majority_share_shuffle { + role: SECOND_NON_AGGREGATOR + first_non_aggregator_duchy_id: "worker1" + second_non_aggregator_duchy_id: "worker2" + aggregator_duchy_id: "aggregator" +} \ No newline at end of file diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/BUILD.bazel new file mode 100644 index 00000000000..4ad35e147f9 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/BUILD.bazel @@ -0,0 +1,42 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@wfa_common_jvm//build:defs.bzl", "test_target") +load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") +load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") +load("//src/main/docker:macros.bzl", "java_image") + +package( + default_testonly = True, #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + default_visibility = [ + test_target(":__pkg__"), + "//src/main/kotlin/org/wfanet/measurement/duchy/testing:__pkg__", + "//src/test/kotlin/org/wfanet/measurement/integration:__subpackages__", + ], +) + +kt_jvm_library( + name = "s3_herald_daemon", + srcs = ["S3HeraldDaemon.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:herald_daemon", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/aws/s3", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + ], +) + +java_binary( + name = "S3HeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.aws.daemon.herald.S3HeraldDaemonKt", + runtime_deps = [ + ":s3_herald_daemon", + "//imports/java/software/amazon/awssdk/sts", + ], +) + +java_image( + name = "s3_herald_daemon_image", + binary = ":S3HeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.aws.daemon.herald.S3HeraldDaemonKt", + target_compatible_with = DISTROLESS_JAVA, + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/S3HeraldDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/S3HeraldDaemon.kt new file mode 100644 index 00000000000..3c7261cf140 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/herald/S3HeraldDaemon.kt @@ -0,0 +1,38 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.aws.daemon.herald + +import org.wfanet.measurement.aws.s3.S3Flags +import org.wfanet.measurement.aws.s3.S3StorageClient +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.duchy.deploy.common.daemon.herald.HeraldDaemon +import picocli.CommandLine + +@CommandLine.Command( + name = "S3HeraldDaemon", + description = ["S3 herald daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class S3HeraldDaemon : HeraldDaemon() { + @CommandLine.Mixin private lateinit var s3Flags: S3Flags + + override fun run() { + val storageClient = S3StorageClient.fromFlags(s3Flags) + run(storageClient) + } +} + +fun main(args: Array) = commandLineMain(S3HeraldDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/BUILD.bazel new file mode 100644 index 00000000000..0e6ed5feac9 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/BUILD.bazel @@ -0,0 +1,42 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@wfa_common_jvm//build:defs.bzl", "test_target") +load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") +load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") +load("//src/main/docker:macros.bzl", "java_image") + +package( + default_testonly = True, #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + default_visibility = [ + test_target(":__pkg__"), + "//src/main/kotlin/org/wfanet/measurement/duchy/testing:__pkg__", + "//src/test/kotlin/org/wfanet/measurement/integration:__subpackages__", + ], +) + +kt_jvm_library( + name = "s3_honest_majority_share_shuffle_mill_daemon", + srcs = ["S3HonestMajorityShareShuffleMillDaemon.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle:honest_majority_share_shuffle_mill_daemon", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/aws/s3", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + ], +) + +java_binary( + name = "S3HonestMajorityShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.aws.daemon.mill.shareshuffle.S3HonestMajorityShareShuffleMillDaemonKt", + runtime_deps = [ + ":s3_honest_majority_share_shuffle_mill_daemon", + "//imports/java/software/amazon/awssdk/sts", + ], +) + +java_image( + name = "s3_honest_majority_share_shuffle_mill_daemon_image", + binary = ":S3HonestMajorityShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.aws.daemon.mill.shareshuffle.S3HonestMajorityShareShuffleMillDaemonKt", + target_compatible_with = DISTROLESS_JAVA, + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/S3HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/S3HonestMajorityShareShuffleMillDaemon.kt new file mode 100644 index 00000000000..796769826f8 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/aws/daemon/mill/shareshuffle/S3HonestMajorityShareShuffleMillDaemon.kt @@ -0,0 +1,38 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.aws.daemon.mill.shareshuffle + +import org.wfanet.measurement.aws.s3.S3Flags +import org.wfanet.measurement.aws.s3.S3StorageClient +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle.HonestMajorityShareShuffleMillDaemon +import picocli.CommandLine + +@CommandLine.Command( + name = "S3HonestMajorityShareShuffleMillDaemon", + description = ["Honest Majority Share Shuffle Mill daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class S3HonestMajorityShareShuffleMillDaemon : HonestMajorityShareShuffleMillDaemon() { + @CommandLine.Mixin private lateinit var s3Flags: S3Flags + + override fun run() { + val storageClient = S3StorageClient.fromFlags(s3Flags) + run(storageClient) + } +} + +fun main(args: Array) = commandLineMain(S3HonestMajorityShareShuffleMillDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/ServiceFlags.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/ServiceFlags.kt index 75158ad12de..63c12852dfb 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/ServiceFlags.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/ServiceFlags.kt @@ -91,3 +91,26 @@ class SystemApiFlags { var certHost: String? = null private set } + +// TODO(@renjiez): Add Certificate Service in kingdom system api. +class KingdomPublicApiFlags { + @CommandLine.Option( + names = ["--kingdom-public-api-target"], + description = ["gRPC target (authority) of the Kingdom public API server"], + required = true, + ) + lateinit var target: String + private set + + @CommandLine.Option( + names = ["--kingdom-public-api-cert-host"], + description = + [ + "Expected hostname (DNS-ID) in the Kingdom public API server's TLS certificate.", + "This overrides derivation of the TLS DNS-ID from --kingdom-public-api-target.", + ], + required = false, + ) + var certHost: String? = null + private set +} diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel index 3ab12b8e9a8..2e31574aefc 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/BUILD.bazel @@ -2,10 +2,12 @@ load("@rules_java//java:defs.bzl", "java_binary") load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") load("//src/main/docker:macros.bzl", "java_image") +package(default_testonly = True) #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + kt_jvm_library( name = "herald_daemon", - testonly = True, #TODO(@renjiez): delete testonly after removing PrivateKeyClientPlaceholder. srcs = ["HeraldDaemon.kt"], + visibility = ["//src/main/kotlin/org/wfanet/measurement/duchy/deploy:__subpackages__"], deps = [ "//src/main/kotlin/org/wfanet/measurement/common/identity", "//src/main/kotlin/org/wfanet/measurement/duchy/daemon/herald", @@ -26,20 +28,29 @@ kt_jvm_library( ], ) +kt_jvm_library( + name = "forwarded_storage_herald_daemon", + srcs = ["ForwardedStorageHeraldDaemon.kt"], + deps = [ + ":herald_daemon", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/forwarded", + ], +) + java_binary( - name = "HeraldDaemon", - testonly = True, #TODO(@renjiez): delete testonly after removing PrivateKeyClientPlaceholder. - main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.herald.HeraldDaemonKt", + name = "ForwardedStorageHeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.herald.ForwardedStorageHeraldDaemonKt", runtime_deps = [ - ":herald_daemon", + ":forwarded_storage_herald_daemon", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/logging", ], ) java_image( - name = "herald_daemon_image", - testonly = True, #TODO(@renjiez): delete testonly after removing PrivateKeyClientPlaceholder. - binary = ":HeraldDaemon", - main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.herald.HeraldDaemonKt", + name = "forwarded_storage_herald_daemon_image", + binary = ":ForwardedStorageHeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.herald.ForwardedStorageHeraldDaemonKt", visibility = ["//src:docker_image_deployment"], ) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/ForwardedStorageHeraldDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/ForwardedStorageHeraldDaemon.kt new file mode 100644 index 00000000000..71b2d3bf822 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/ForwardedStorageHeraldDaemon.kt @@ -0,0 +1,35 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.common.daemon.herald + +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.storage.forwarded.ForwardedStorageFromFlags +import picocli.CommandLine + +@CommandLine.Command( + name = "ForwardedStorageHeraldDaemon", + description = ["Forwarded storage herald daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class ForwardedStorageHeraldDaemon : HeraldDaemon() { + @CommandLine.Mixin private lateinit var forwardedStorageFlags: ForwardedStorageFromFlags.Flags + + override fun run() { + run(ForwardedStorageFromFlags(forwardedStorageFlags, flags.tlsFlags).storageClient) + } +} + +fun main(args: Array) = commandLineMain(ForwardedStorageHeraldDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt index 4a0310424f0..4e26c97ce78 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt @@ -15,7 +15,8 @@ package org.wfanet.measurement.duchy.deploy.common.daemon.herald import com.google.crypto.tink.Aead -import com.google.crypto.tink.KeyTemplates +import com.google.crypto.tink.BinaryKeysetReader +import com.google.crypto.tink.CleartextKeysetHandle import com.google.crypto.tink.KeysetHandle import com.google.crypto.tink.aead.AeadConfig import io.grpc.Channel @@ -24,7 +25,6 @@ import java.time.Clock import java.time.Duration import kotlin.properties.Delegates import kotlinx.coroutines.runBlocking -import org.wfanet.measurement.common.commandLineMain import org.wfanet.measurement.common.crypto.SigningCerts import org.wfanet.measurement.common.crypto.tink.TinkKeyStorageProvider import org.wfanet.measurement.common.crypto.tink.testing.FakeKmsClient @@ -44,13 +44,13 @@ import org.wfanet.measurement.duchy.storage.TinkKeyStore import org.wfanet.measurement.internal.duchy.ComputationsGrpcKt.ComputationsCoroutineStub import org.wfanet.measurement.internal.duchy.ContinuationTokensGrpcKt.ContinuationTokensCoroutineStub import org.wfanet.measurement.internal.duchy.config.ProtocolsSetupConfig -import org.wfanet.measurement.storage.testing.InMemoryStorageClient +import org.wfanet.measurement.storage.StorageClient import org.wfanet.measurement.system.v1alpha.Computation import org.wfanet.measurement.system.v1alpha.ComputationParticipantsGrpcKt.ComputationParticipantsCoroutineStub as SystemComputationParticipantsCoroutineStub import org.wfanet.measurement.system.v1alpha.ComputationsGrpcKt.ComputationsCoroutineStub as SystemComputationsCoroutineStub import picocli.CommandLine -private class Flags { +class HeraldFlags { @CommandLine.Mixin lateinit var duchy: CommonDuchyFlags private set @@ -83,6 +83,12 @@ private class Flags { lateinit var protocolsSetupConfig: File private set + @CommandLine.Option( + names = ["--key-encryption-key-file"], + description = ["The key encryption key file (binary format) used for private key store."], + ) + var keyEncryptionKeyTinkFile: File? = null + @CommandLine.Option( names = ["--deletable-computation-state"], description = @@ -104,72 +110,81 @@ private class Flags { private set } -@CommandLine.Command( - name = "HeraldDaemon", - mixinStandardHelpOptions = true, - showDefaultValues = true, -) -private fun run(@CommandLine.Mixin flags: Flags) { - val clientCerts = - SigningCerts.fromPemFiles( - certificateFile = flags.tlsFlags.certFile, - privateKeyFile = flags.tlsFlags.privateKeyFile, - trustedCertCollectionFile = flags.tlsFlags.certCollectionFile, - ) - - val systemServiceChannel = - buildMutualTlsChannel(flags.systemApiFlags.target, clientCerts, flags.systemApiFlags.certHost) - .withShutdownTimeout(flags.channelShutdownTimeout) - .withVerboseLogging(flags.verboseGrpcClientLogging) - val systemComputationsClient = - SystemComputationsCoroutineStub(systemServiceChannel).withDuchyId(flags.duchy.duchyName) - val systemComputationParticipantsClient = - SystemComputationParticipantsCoroutineStub(systemServiceChannel) - .withDuchyId(flags.duchy.duchyName) - - val internalComputationsChannel: Channel = - buildMutualTlsChannel( - flags.computationsServiceFlags.target, - clientCerts, - flags.computationsServiceFlags.certHost, +abstract class HeraldDaemon : Runnable { + @CommandLine.Mixin + protected lateinit var flags: HeraldFlags + private set + + @CommandLine.Command( + name = "HeraldDaemon", + mixinStandardHelpOptions = true, + showDefaultValues = true, + ) + protected fun run(storageClient: StorageClient) { + val clientCerts = + SigningCerts.fromPemFiles( + certificateFile = flags.tlsFlags.certFile, + privateKeyFile = flags.tlsFlags.privateKeyFile, + trustedCertCollectionFile = flags.tlsFlags.certCollectionFile, ) - .withShutdownTimeout(flags.channelShutdownTimeout) - .withDefaultDeadline(flags.computationsServiceFlags.defaultDeadlineDuration) - .withVerboseLogging(flags.verboseGrpcClientLogging) - val internalComputationsClient = ComputationsCoroutineStub(internalComputationsChannel) - - val continuationTokenClient = ContinuationTokensCoroutineStub(internalComputationsChannel) - val continuationTokenManager = ContinuationTokenManager(continuationTokenClient) - // This will be the name of the pod when deployed to Kubernetes. - val heraldId = System.getenv("HOSTNAME") - - // TODO(@renjiez): Use real PrivateKeyStore when enabling HMSS. - AeadConfig.register() - val keyUri = "fake-kms://kek" - val privateKeyHandle = KeysetHandle.generateNew(KeyTemplates.get("AES128_GCM")) - val aead = privateKeyHandle.getPrimitive(Aead::class.java) - val fakeKmsClient = FakeKmsClient().also { it.setAead(keyUri, aead) } - val privateKeyStore = - TinkKeyStorageProvider(fakeKmsClient) - .makeKmsPrivateKeyStore(TinkKeyStore(InMemoryStorageClient()), keyUri) - - val herald = - Herald( - heraldId = heraldId, - duchyId = flags.duchy.duchyName, - internalComputationsClient = internalComputationsClient, - systemComputationsClient = systemComputationsClient, - systemComputationParticipantClient = systemComputationParticipantsClient, - privateKeyStore = privateKeyStore, - continuationTokenManager = continuationTokenManager, - protocolsSetupConfig = - flags.protocolsSetupConfig.reader().use { - parseTextProto(it, ProtocolsSetupConfig.getDefaultInstance()) - }, - clock = Clock.systemUTC(), - deletableComputationStates = flags.deletableComputationStates, - ) - runBlocking { herald.continuallySyncStatuses() } -} -fun main(args: Array) = commandLineMain(::run, args) + val systemServiceChannel = + buildMutualTlsChannel(flags.systemApiFlags.target, clientCerts, flags.systemApiFlags.certHost) + .withShutdownTimeout(flags.channelShutdownTimeout) + .withVerboseLogging(flags.verboseGrpcClientLogging) + val systemComputationsClient = + SystemComputationsCoroutineStub(systemServiceChannel).withDuchyId(flags.duchy.duchyName) + val systemComputationParticipantsClient = + SystemComputationParticipantsCoroutineStub(systemServiceChannel) + .withDuchyId(flags.duchy.duchyName) + + val internalComputationsChannel: Channel = + buildMutualTlsChannel( + flags.computationsServiceFlags.target, + clientCerts, + flags.computationsServiceFlags.certHost, + ) + .withShutdownTimeout(flags.channelShutdownTimeout) + .withDefaultDeadline(flags.computationsServiceFlags.defaultDeadlineDuration) + .withVerboseLogging(flags.verboseGrpcClientLogging) + val internalComputationsClient = ComputationsCoroutineStub(internalComputationsChannel) + + val continuationTokenClient = ContinuationTokensCoroutineStub(internalComputationsChannel) + val continuationTokenManager = ContinuationTokenManager(continuationTokenClient) + // This will be the name of the pod when deployed to Kubernetes. + val heraldId = System.getenv("HOSTNAME") + + val privateKeyStore = + flags.keyEncryptionKeyTinkFile?.let { file -> + val keyUri = FakeKmsClient.KEY_URI_PREFIX + "kek" + + val keysetHandle: KeysetHandle = + file.inputStream().use { input -> + CleartextKeysetHandle.read(BinaryKeysetReader.withInputStream(input)) + } + AeadConfig.register() + val aead = keysetHandle.getPrimitive(Aead::class.java) + val fakeKmsClient = FakeKmsClient().also { it.setAead(keyUri, aead) } + TinkKeyStorageProvider(fakeKmsClient) + .makeKmsPrivateKeyStore(TinkKeyStore(storageClient), keyUri) + } + + val herald = + Herald( + heraldId = heraldId, + duchyId = flags.duchy.duchyName, + internalComputationsClient = internalComputationsClient, + systemComputationsClient = systemComputationsClient, + systemComputationParticipantClient = systemComputationParticipantsClient, + privateKeyStore = privateKeyStore, + continuationTokenManager = continuationTokenManager, + protocolsSetupConfig = + flags.protocolsSetupConfig.reader().use { + parseTextProto(it, ProtocolsSetupConfig.getDefaultInstance()) + }, + clock = Clock.systemUTC(), + deletableComputationStates = flags.deletableComputationStates, + ) + runBlocking { herald.continuallySyncStatuses() } + } +} diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel new file mode 100644 index 00000000000..cb0daa2d312 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel @@ -0,0 +1,66 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") +load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") +load("//src/main/docker:macros.bzl", "java_image") + +package(default_testonly = True) #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + +kt_jvm_library( + name = "honest_majority_share_shuffle_mill_daemon", + srcs = [ + "HonestMajorityShareShuffleMillDaemon.kt", + "HonestMajorityShareShuffleMillFlags.kt", + ], + visibility = ["//src/main/kotlin/org/wfanet/measurement/duchy/deploy:__subpackages__"], + deps = [ + "//imports/java/io/opentelemetry/exporter/otlp", + "//imports/java/io/opentelemetry/sdk", + "//imports/java/io/opentelemetry/sdk:common", + "//imports/java/io/opentelemetry/sdk:metrics", + "//imports/java/io/opentelemetry/semconv", + "//src/main/kotlin/org/wfanet/measurement/common/identity", + "//src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle:honest_majority_share_shuffle_mill", + "//src/main/kotlin/org/wfanet/measurement/duchy/db/computation", + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common:flags", + "//src/main/kotlin/org/wfanet/measurement/duchy/storage:tink_key_store", + "//src/main/proto/wfa/measurement/internal/duchy:computations_service_kt_jvm_grpc_proto", + "//src/main/proto/wfa/measurement/internal/duchy/protocol:honest_majority_share_shuffle_kt_jvm_proto", + "//src/main/proto/wfa/measurement/system/v1alpha:computation_control_service_kt_jvm_grpc_proto", + "@wfa_common_jvm//imports/java/com/google/common:guava", + "@wfa_common_jvm//imports/java/com/google/protobuf", + "@wfa_common_jvm//imports/java/io/grpc:api", + "@wfa_common_jvm//imports/java/io/opentelemetry/api", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines:core", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/crypto/tink", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/crypto/tink/testing", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/grpc", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/identity", + ], +) + +kt_jvm_library( + name = "forwarded_storage_honest_majority_share_shuffle_mill_daemon", + srcs = ["ForwardedStorageHonestMajorityShareShuffleDaemon.kt"], + deps = [ + ":honest_majority_share_shuffle_mill_daemon", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/forwarded", + ], +) + +java_binary( + name = "ForwardedStorageHonestMajorityShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle.ForwardedStorageHonestMajorityShareShuffleDaemonKt", + runtime_deps = [":forwarded_storage_honest_majority_share_shuffle_mill_daemon"], +) + +java_image( + name = "forwarded_storage_honest_majority_share_shuffle_mill_daemon_image", + binary = ":ForwardedStorageHonestMajorityShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle.ForwardedStorageHonestMajorityShareShuffleDaemonKt", + target_compatible_with = DISTROLESS_JAVA, + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/ForwardedStorageHonestMajorityShareShuffleDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/ForwardedStorageHonestMajorityShareShuffleDaemon.kt new file mode 100644 index 00000000000..1973bff59ac --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/ForwardedStorageHonestMajorityShareShuffleDaemon.kt @@ -0,0 +1,37 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle + +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.storage.forwarded.ForwardedStorageFromFlags +import picocli.CommandLine + +@CommandLine.Command( + name = "ForwardedStorageHonestMajorityShareShuffleMillDaemon", + description = ["HonestMajorityShareShuffle Mill daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class ForwardedStorageHonestMajorityShareShuffleMillDaemon : + HonestMajorityShareShuffleMillDaemon() { + @CommandLine.Mixin private lateinit var forwardedStorageFlags: ForwardedStorageFromFlags.Flags + + override fun run() { + run(ForwardedStorageFromFlags(forwardedStorageFlags, flags.tlsFlags).storageClient) + } +} + +fun main(args: Array) = + commandLineMain(ForwardedStorageHonestMajorityShareShuffleMillDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt new file mode 100644 index 00000000000..b2da9538670 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt @@ -0,0 +1,250 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle + +import com.google.crypto.tink.Aead +import com.google.crypto.tink.BinaryKeysetReader +import com.google.crypto.tink.CleartextKeysetHandle +import com.google.crypto.tink.KeysetHandle +import com.google.crypto.tink.aead.AeadConfig +import com.google.protobuf.ByteString +import io.grpc.Channel +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.metrics.Aggregation +import io.opentelemetry.sdk.metrics.InstrumentSelector +import io.opentelemetry.sdk.metrics.InstrumentType +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.View +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.semconv.ResourceAttributes +import java.time.Clock +import java.time.Duration +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.wfanet.measurement.api.v2alpha.CertificatesGrpcKt.CertificatesCoroutineStub +import org.wfanet.measurement.common.crypto.SigningCerts +import org.wfanet.measurement.common.crypto.SigningKeyHandle +import org.wfanet.measurement.common.crypto.readCertificate +import org.wfanet.measurement.common.crypto.readPrivateKey +import org.wfanet.measurement.common.crypto.tink.TinkKeyStorageProvider +import org.wfanet.measurement.common.crypto.tink.testing.FakeKmsClient +import org.wfanet.measurement.common.grpc.buildMutualTlsChannel +import org.wfanet.measurement.common.grpc.withDefaultDeadline +import org.wfanet.measurement.common.grpc.withShutdownTimeout +import org.wfanet.measurement.common.identity.DuchyInfo +import org.wfanet.measurement.common.identity.withDuchyId +import org.wfanet.measurement.common.logAndSuppressExceptionSuspend +import org.wfanet.measurement.common.parseTextProto +import org.wfanet.measurement.common.throttler.MinimumIntervalThrottler +import org.wfanet.measurement.duchy.daemon.mill.Certificate +import org.wfanet.measurement.duchy.daemon.mill.shareshuffle.HonestMajorityShareShuffleMill +import org.wfanet.measurement.duchy.daemon.mill.shareshuffle.crypto.JniHonestMajorityShareShuffleCryptor +import org.wfanet.measurement.duchy.db.computation.ComputationDataClients +import org.wfanet.measurement.duchy.storage.TinkKeyStore +import org.wfanet.measurement.internal.duchy.ComputationStatsGrpcKt.ComputationStatsCoroutineStub +import org.wfanet.measurement.internal.duchy.ComputationsGrpcKt.ComputationsCoroutineStub +import org.wfanet.measurement.internal.duchy.config.HonestMajorityShareShuffleSetupConfig +import org.wfanet.measurement.storage.StorageClient +import org.wfanet.measurement.system.v1alpha.ComputationControlGrpcKt.ComputationControlCoroutineStub +import org.wfanet.measurement.system.v1alpha.ComputationLogEntriesGrpcKt.ComputationLogEntriesCoroutineStub as SystemComputationLogEntriesCoroutineStub +import org.wfanet.measurement.system.v1alpha.ComputationParticipantsGrpcKt.ComputationParticipantsCoroutineStub as SystemComputationParticipantsCoroutineStub +import org.wfanet.measurement.system.v1alpha.ComputationsGrpcKt.ComputationsCoroutineStub as SystemComputationsCoroutineStub +import picocli.CommandLine + +abstract class HonestMajorityShareShuffleMillDaemon : Runnable { + @CommandLine.Mixin + protected lateinit var flags: HonestMajorityShareShuffleMillFlags + private set + + protected fun run(storageClient: StorageClient) { + DuchyInfo.initializeFromFlags(flags.duchyInfoFlags) + val duchyName = flags.duchy.duchyName + + val clientCerts = + SigningCerts.fromPemFiles( + certificateFile = flags.tlsFlags.certFile, + privateKeyFile = flags.tlsFlags.privateKeyFile, + trustedCertCollectionFile = flags.tlsFlags.certCollectionFile, + ) + + val computationsServiceChannel: Channel = + buildMutualTlsChannel( + flags.computationsServiceFlags.target, + clientCerts, + flags.computationsServiceFlags.certHost, + ) + .withShutdownTimeout(flags.channelShutdownTimeout) + .withDefaultDeadline(flags.computationsServiceFlags.defaultDeadlineDuration) + val dataClients = + ComputationDataClients( + ComputationsCoroutineStub(computationsServiceChannel).withDuchyId(duchyName), + storageClient, + ) + + val computationControlClientMap = + DuchyInfo.entries + .filterKeys { it != duchyName } + .mapValues { (duchyId, entry) -> + ComputationControlCoroutineStub( + buildMutualTlsChannel( + flags.computationControlServiceTargets.getValue(duchyId), + clientCerts, + entry.computationControlServiceCertHost, + ) + .withShutdownTimeout(flags.channelShutdownTimeout) + ) + .withDuchyId(duchyName) + } + + val systemApiChannel = + buildMutualTlsChannel(flags.systemApiFlags.target, clientCerts, flags.systemApiFlags.certHost) + .withShutdownTimeout(flags.channelShutdownTimeout) + + val publicApiChannel = + buildMutualTlsChannel(flags.publicApiFlags.target, clientCerts, flags.publicApiFlags.certHost) + .withShutdownTimeout(flags.channelShutdownTimeout) + + val systemComputationsClient = + SystemComputationsCoroutineStub(systemApiChannel).withDuchyId(duchyName) + val systemComputationParticipantsClient = + SystemComputationParticipantsCoroutineStub(systemApiChannel).withDuchyId(duchyName) + val systemComputationLogEntriesClient = + SystemComputationLogEntriesCoroutineStub(systemApiChannel).withDuchyId(duchyName) + + val publicCertificatesClient = + CertificatesCoroutineStub(publicApiChannel).withDuchyId(duchyName) + + val computationStatsClient = ComputationStatsCoroutineStub(computationsServiceChannel) + + val csX509Certificate = + flags.csCertificateDerFile.inputStream().use { input -> readCertificate(input) } + val csCertificate = Certificate(flags.csCertificateName, csX509Certificate) + // TODO: Read from a KMS-encrypted store instead. + val csSigningKey = + SigningKeyHandle( + csX509Certificate, + flags.csPrivateKeyDerFile.inputStream().use { input -> + readPrivateKey(ByteString.readFrom(input), csX509Certificate.publicKey.algorithm) + }, + ) + + // This will be the name of the pod when deployed to Kubernetes. Note that the millId is + // included in mill logs to help debugging. + val millId = System.getenv("HOSTNAME") + + val privateKeyStore = + flags.keyEncryptionKeyTinkFile?.let { file -> + val keyUri = FakeKmsClient.KEY_URI_PREFIX + "kek" + + val keysetHandle: KeysetHandle = + file.inputStream().use { input -> + CleartextKeysetHandle.read(BinaryKeysetReader.withInputStream(input)) + } + AeadConfig.register() + val aead = keysetHandle.getPrimitive(Aead::class.java) + val fakeKmsClient = FakeKmsClient().also { it.setAead(keyUri, aead) } + TinkKeyStorageProvider(fakeKmsClient) + .makeKmsPrivateKeyStore(TinkKeyStore(storageClient), keyUri) + } + + val openTelemetry: OpenTelemetry = + if (flags.openTelemetryOptions == null) { + GlobalOpenTelemetry.get() + } else { + val endpoint = flags.openTelemetryOptions!!.otelExporterOtlpEndpoint + val serviceName = flags.openTelemetryOptions!!.otelServiceName + val resource: Resource = + Resource.getDefault() + .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, serviceName))) + val meterProvider = + SdkMeterProvider.builder() + .setResource(resource) + .registerMetricReader( + PeriodicMetricReader.builder( + OtlpGrpcMetricExporter.builder() + .setTimeout(Duration.ofSeconds(30L)) + .setEndpoint(endpoint) + .build() + ) + .setInterval(Duration.ofSeconds(60L)) + .build() + ) + .registerView( + InstrumentSelector.builder().setType(InstrumentType.HISTOGRAM).build(), + View.builder() + .setAggregation( + Aggregation.explicitBucketHistogram( + listOf( + 1000.0, + 2000.0, + 4000.0, + 8000.0, + 16000.0, + 32000.0, + 64000.0, + 128000.0, + 256000.0, + 512000.0, + 1024000.0, + ) + ) + ) + .build(), + ) + .build() + OpenTelemetrySdk.builder().setMeterProvider(meterProvider).build() + } + + val mill = + HonestMajorityShareShuffleMill( + millId = millId, + duchyId = flags.duchy.duchyName, + signingKey = csSigningKey, + consentSignalCert = csCertificate, + trustedCertificates = flags.tlsFlags.signingCerts.trustedCertificates, + dataClients = dataClients, + systemComputationParticipantsClient = systemComputationParticipantsClient, + systemComputationsClient = systemComputationsClient, + systemComputationLogEntriesClient = systemComputationLogEntriesClient, + computationStatsClient = computationStatsClient, + privateKeyStore = privateKeyStore, + certificateClient = publicCertificatesClient, + workerStubs = computationControlClientMap, + cryptoWorker = JniHonestMajorityShareShuffleCryptor(), + protocolSetupConfig = + flags.protocolsSetupConfig.reader().use { + parseTextProto(it, HonestMajorityShareShuffleSetupConfig.getDefaultInstance()) + }, + workLockDuration = flags.workLockDuration, + openTelemetry = openTelemetry, + requestChunkSizeBytes = flags.requestChunkSizeBytes, + ) + + runBlocking { + withContext(CoroutineName("Mill $millId")) { + val throttler = MinimumIntervalThrottler(Clock.systemUTC(), flags.pollingInterval) + throttler.loopOnReady { + logAndSuppressExceptionSuspend { mill.pollAndProcessNextComputation() } + } + } + } + } +} diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt new file mode 100644 index 00000000000..a6bcbf0a14d --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt @@ -0,0 +1,150 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle + +import java.io.File +import java.time.Duration +import kotlin.properties.Delegates +import org.wfanet.measurement.common.grpc.TlsFlags +import org.wfanet.measurement.common.identity.DuchyInfoFlags +import org.wfanet.measurement.duchy.deploy.common.CommonDuchyFlags +import org.wfanet.measurement.duchy.deploy.common.ComputationsServiceFlags +import org.wfanet.measurement.duchy.deploy.common.KingdomPublicApiFlags +import org.wfanet.measurement.duchy.deploy.common.SystemApiFlags +import picocli.CommandLine + +class HonestMajorityShareShuffleMillFlags { + @CommandLine.Mixin + lateinit var duchy: CommonDuchyFlags + private set + + @CommandLine.Mixin + lateinit var tlsFlags: TlsFlags + private set + + @CommandLine.Mixin + lateinit var duchyInfoFlags: DuchyInfoFlags + private set + + @CommandLine.Option( + names = ["--duchy-computation-control-target"], + description = ["Key-value pair of Duchy ID to ComputationControl service target."], + required = true, + ) + lateinit var computationControlServiceTargets: Map + private set + + @CommandLine.Option( + names = ["--channel-shutdown-timeout"], + defaultValue = "3s", + description = ["How long to allow for the gRPC channel to shutdown."], + ) + lateinit var channelShutdownTimeout: Duration + private set + + @CommandLine.Option( + names = ["--polling-interval"], + defaultValue = "2s", + description = ["How long to sleep before polling the computation queue again if it is empty."], + ) + lateinit var pollingInterval: Duration + private set + + @CommandLine.Option( + names = ["--work-lock-duration"], + defaultValue = "5m", + description = ["How long to hold work locks."], + ) + lateinit var workLockDuration: Duration + private set + + @CommandLine.Mixin + lateinit var systemApiFlags: SystemApiFlags + private set + + @CommandLine.Mixin + lateinit var publicApiFlags: KingdomPublicApiFlags + private set + + @CommandLine.Mixin + lateinit var computationsServiceFlags: ComputationsServiceFlags + private set + + @set:CommandLine.Option( + names = ["--bytes-per-chunk"], + description = ["The number of bytes in a chunk when sending rpc result to other duchy."], + defaultValue = "32768", // 32 KiB. See https://github.com/grpc/grpc.github.io/issues/371. + ) + var requestChunkSizeBytes by Delegates.notNull() + private set + + @CommandLine.Option( + names = ["--consent-signaling-certificate-resource-name"], + description = ["The resource name of the duchy's consent signaling certificate."], + required = true, + ) + lateinit var csCertificateName: String + private set + + @CommandLine.Option( + names = ["--consent-signaling-private-key-der-file"], + description = ["The duchy's consent signaling private key (DER format) file."], + required = true, + ) + lateinit var csPrivateKeyDerFile: File + private set + + @CommandLine.Option( + names = ["--consent-signaling-certificate-der-file"], + description = ["The duchy's consent signaling certificate (DER format) file."], + required = true, + ) + lateinit var csCertificateDerFile: File + private set + + @CommandLine.Option( + names = ["--protocols-setup-config"], + description = ["ProtocolsSetupConfig proto message in text format."], + required = true, + ) + lateinit var protocolsSetupConfig: File + private set + + @CommandLine.Option( + names = ["--key-encryption-key-file"], + description = ["The key encryption key file (binary format) used for private key store."], + ) + var keyEncryptionKeyTinkFile: File? = null + + @CommandLine.ArgGroup(exclusive = false) var openTelemetryOptions: OpenTelemetryOptions? = null + // All options here must be present or none of them must be present. + class OpenTelemetryOptions { + @CommandLine.Option( + names = ["--otel-exporter-otlp-endpoint"], + description = ["Endpoint for OpenTelemetry Collector."], + required = true, + ) + lateinit var otelExporterOtlpEndpoint: String + private set + + @CommandLine.Option( + names = ["--otel-service-name"], + description = ["Service name to label duchy metrics with."], + required = true, + ) + lateinit var otelServiceName: String + private set + } +} diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/BUILD.bazel new file mode 100644 index 00000000000..35b6940ef95 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/BUILD.bazel @@ -0,0 +1,42 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@wfa_common_jvm//build:defs.bzl", "test_target") +load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") +load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") +load("//src/main/docker:macros.bzl", "java_image") + +package( + default_testonly = True, #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + default_visibility = [ + test_target(":__pkg__"), + "//src/main/kotlin/org/wfanet/measurement/duchy/testing:__pkg__", + "//src/test/kotlin/org/wfanet/measurement/integration:__subpackages__", + ], +) + +kt_jvm_library( + name = "gcs_herald_daemon", + srcs = ["GcsHeraldDaemon.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald:herald_daemon", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/gcs", + ], +) + +java_binary( + name = "GcsHeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.daemon.herald.GcsHeraldDaemonKt", + runtime_deps = [ + ":gcs_herald_daemon", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/logging", + ], +) + +java_image( + name = "gcs_herald_daemon_image", + binary = ":GcsHeraldDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.daemon.herald.GcsHeraldDaemonKt", + target_compatible_with = DISTROLESS_JAVA, + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/GcsHeraldDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/GcsHeraldDaemon.kt new file mode 100644 index 00000000000..3cb8ea0fa6b --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/herald/GcsHeraldDaemon.kt @@ -0,0 +1,38 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.gcloud.daemon.herald + +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.duchy.deploy.common.daemon.herald.HeraldDaemon +import org.wfanet.measurement.gcloud.gcs.GcsFromFlags +import org.wfanet.measurement.gcloud.gcs.GcsStorageClient +import picocli.CommandLine + +@CommandLine.Command( + name = "GcsHeraldDaemon", + description = ["Gcs herald daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class GcsHeraldDaemon : HeraldDaemon() { + @CommandLine.Mixin private lateinit var gcsFlags: GcsFromFlags.Flags + + override fun run() { + val gcs = GcsFromFlags(gcsFlags) + run(GcsStorageClient.fromFlags(gcs)) + } +} + +fun main(args: Array) = commandLineMain(GcsHeraldDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/BUILD.bazel new file mode 100644 index 00000000000..72fe059b997 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@wfa_common_jvm//build:defs.bzl", "test_target") +load("@wfa_rules_kotlin_jvm//kotlin:defs.bzl", "kt_jvm_library") +load("//build/platforms:constraints.bzl", "DISTROLESS_JAVA") +load("//src/main/docker:macros.bzl", "java_image") + +package( + default_testonly = True, #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used. + default_visibility = [ + test_target(":__pkg__"), + "//src/main/kotlin/org/wfanet/measurement/duchy/testing:__pkg__", + "//src/test/kotlin/org/wfanet/measurement/integration:__subpackages__", + ], +) + +kt_jvm_library( + name = "gcs_honest_majority_share_shuffle_mill_daemon", + srcs = ["GcsHonestMajorityShareShuffleMillDaemon.kt"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle:honest_majority_share_shuffle_mill_daemon", + "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/spanner/computation", + "@wfa_common_jvm//imports/java/picocli", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/gcs", + ], +) + +java_binary( + name = "GcsHonestMajoriyShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.daemon.mill.shareshuffle.GcsHonestMajorityShareShuffleMillDaemonKt", + runtime_deps = [ + ":gcs_honest_majority_share_shuffle_mill_daemon", + "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/logging", + ], +) + +java_image( + name = "gcs_honest_majority_share_shuffle_mill_daemon_image", + binary = ":GcsHonestMajoriyShareShuffleMillDaemon", + main_class = "org.wfanet.measurement.duchy.deploy.gcloud.daemon.mill.shareshuffle.GcsHonestMajorityShareShuffleMillDaemonKt", + target_compatible_with = DISTROLESS_JAVA, + visibility = ["//src:docker_image_deployment"], +) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/GcsHonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/GcsHonestMajorityShareShuffleMillDaemon.kt new file mode 100644 index 00000000000..89279b41b4d --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/gcloud/daemon/mill/shareshuffle/GcsHonestMajorityShareShuffleMillDaemon.kt @@ -0,0 +1,38 @@ +// Copyright 2024 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.wfanet.measurement.duchy.deploy.gcloud.daemon.mill.shareshuffle + +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.duchy.deploy.common.daemon.mill.shareshuffle.HonestMajorityShareShuffleMillDaemon +import org.wfanet.measurement.gcloud.gcs.GcsFromFlags +import org.wfanet.measurement.gcloud.gcs.GcsStorageClient +import picocli.CommandLine + +@CommandLine.Command( + name = "GcsHonestMajorityShareShuffleMillDaemon", + description = ["Honest Majority Share Shuffle Mill daemon."], + mixinStandardHelpOptions = true, + showDefaultValues = true, +) +class GcsHonestMajorityShareShuffleMillDaemon : HonestMajorityShareShuffleMillDaemon() { + @CommandLine.Mixin private lateinit var gcsFlags: GcsFromFlags.Flags + + override fun run() { + val gcs = GcsFromFlags(gcsFlags) + run(GcsStorageClient.fromFlags(gcs)) + } +} + +fun main(args: Array) = commandLineMain(GcsHonestMajorityShareShuffleMillDaemon(), args) diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessDuchy.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessDuchy.kt index 3860a0a2f03..adef4caf53c 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessDuchy.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessDuchy.kt @@ -132,7 +132,7 @@ class InProcessDuchy( // TODO(@renjiez): Use real PrivateKeyStore when enabling HMSS. private val privateKeyStore by lazy { - val keyUri = "fake-kms://kek" + val keyUri = FakeKmsClient.KEY_URI_PREFIX + "kek" val privateKeyHandle = KeysetHandle.generateNew(KeyTemplates.get("AES128_GCM")) val aead = privateKeyHandle.getPrimitive(Aead::class.java) val fakeKmsClient = FakeKmsClient().also { it.setAead(keyUri, aead) } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/HmssProtocolConfig.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/HmssProtocolConfig.kt index b1a6b7df062..e48a9aa042d 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/HmssProtocolConfig.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/HmssProtocolConfig.kt @@ -25,10 +25,7 @@ object HmssProtocolConfig { lateinit var protocolConfig: ProtocolConfig.HonestMajorityShareShuffle private set - /** - * Set of external IDs of required Duchies, where the first entry must correspond to the Duchy in - * the aggregator role. - */ + /** Set of external IDs of required Duchies. */ lateinit var requiredExternalDuchyIds: Set private set diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel index 55af673d5b5..2a8a25c90f6 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/BUILD.bazel @@ -9,6 +9,7 @@ kt_jvm_library( deps = [ "//src/main/kotlin/org/wfanet/measurement/common/identity", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:duchy_ids", + "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:hmss_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:ro_llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service:data_services", diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt index f3fc867fb7f..828bd278403 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt @@ -24,6 +24,8 @@ import org.wfanet.measurement.common.identity.DuchyInfo import org.wfanet.measurement.common.identity.DuchyInfoFlags import org.wfanet.measurement.kingdom.deploy.common.DuchyIds import org.wfanet.measurement.kingdom.deploy.common.DuchyIdsFlags +import org.wfanet.measurement.kingdom.deploy.common.HmssProtocolConfig +import org.wfanet.measurement.kingdom.deploy.common.HmssProtocolConfigFlags import org.wfanet.measurement.kingdom.deploy.common.Llv2ProtocolConfig import org.wfanet.measurement.kingdom.deploy.common.Llv2ProtocolConfigFlags import org.wfanet.measurement.kingdom.deploy.common.RoLlv2ProtocolConfig @@ -43,6 +45,8 @@ abstract class KingdomDataServer : Runnable { @CommandLine.Mixin private lateinit var roLlv2ProtocolConfigFlags: RoLlv2ProtocolConfigFlags + @CommandLine.Mixin private lateinit var hmssProtocolConfigFlags: HmssProtocolConfigFlags + @CommandLine.Option( names = ["--known-event-group-metadata-type"], description = @@ -70,6 +74,7 @@ abstract class KingdomDataServer : Runnable { DuchyIds.initializeFromFlags(duchyIdsFlags) Llv2ProtocolConfig.initializeFromFlags(llv2ProtocolConfigFlags) RoLlv2ProtocolConfig.initializeFromFlags(roLlv2ProtocolConfigFlags) + HmssProtocolConfig.initializeFromFlags(hmssProtocolConfigFlags) val services = dataServices.buildDataServices().toList() val server = CommonServer.fromFlags(serverFlags, this::class.simpleName!!, services) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/V2alphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/V2alphaPublicApiServer.kt index 46ff248aab7..309ad129f00 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/V2alphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/V2alphaPublicApiServer.kt @@ -49,6 +49,8 @@ import org.wfanet.measurement.internal.kingdom.PopulationsGrpcKt.PopulationsCoro import org.wfanet.measurement.internal.kingdom.PublicKeysGrpcKt.PublicKeysCoroutineStub as InternalPublicKeysCoroutineStub import org.wfanet.measurement.internal.kingdom.RecurringExchangesGrpcKt.RecurringExchangesCoroutineStub as InternalRecurringExchangesCoroutineStub import org.wfanet.measurement.internal.kingdom.RequisitionsGrpcKt.RequisitionsCoroutineStub as InternalRequisitionsCoroutineStub +import org.wfanet.measurement.kingdom.deploy.common.HmssProtocolConfig +import org.wfanet.measurement.kingdom.deploy.common.HmssProtocolConfigFlags import org.wfanet.measurement.kingdom.deploy.common.Llv2ProtocolConfig import org.wfanet.measurement.kingdom.deploy.common.Llv2ProtocolConfigFlags import org.wfanet.measurement.kingdom.deploy.common.RoLlv2ProtocolConfig @@ -90,11 +92,13 @@ private fun run( @CommandLine.Mixin commonServerFlags: CommonServer.Flags, @CommandLine.Mixin llv2ProtocolConfigFlags: Llv2ProtocolConfigFlags, @CommandLine.Mixin roLlv2ProtocolConfigFlags: RoLlv2ProtocolConfigFlags, + @CommandLine.Mixin hmssProtocolConfigFlags: HmssProtocolConfigFlags, @CommandLine.Mixin v2alphaFlags: V2alphaFlags, @CommandLine.Mixin duchyInfoFlags: DuchyInfoFlags, ) { Llv2ProtocolConfig.initializeFromFlags(llv2ProtocolConfigFlags) RoLlv2ProtocolConfig.initializeFromFlags(roLlv2ProtocolConfigFlags) + HmssProtocolConfig.initializeFromFlags(hmssProtocolConfigFlags) DuchyInfo.initializeFromFlags(duchyInfoFlags) val clientCerts = @@ -151,7 +155,9 @@ private fun run( InternalMeasurementsCoroutineStub(channel), internalDataProvidersStub, v2alphaFlags.directNoiseMechanisms, - v2alphaFlags.reachOnlyLlV2Enabled, + reachOnlyLlV2Enabled = v2alphaFlags.reachOnlyLlV2Enabled, + reachOnlyHmssEnabled = v2alphaFlags.hmssForReachEnabled, + reachAndFrequencyHmssEnabled = v2alphaFlags.hmssForRfEnabled, ) .withPrincipalsFromX509AuthorityKeyIdentifiers(principalLookup) .withApiKeyAuthenticationServerInterceptor(internalApiKeysCoroutineStub), @@ -240,6 +246,30 @@ private class V2alphaFlags { var reachOnlyLlV2Enabled by Delegates.notNull() private set + @set:CommandLine.Option( + names = ["--enable-hmss-for-rf"], + description = + [ + "whether to enable to Honest Majority Share Shuffle protocol for ReachAndFrequency Measurement" + ], + negatable = true, + required = false, + defaultValue = "false", + ) + var hmssForRfEnabled by Delegates.notNull() + private set + + @set:CommandLine.Option( + names = ["--enable-hmss-for-reach"], + description = + ["whether to enable to Honest Majority Share Shuffle protocol for Reach Measurement"], + negatable = true, + required = false, + defaultValue = "false", + ) + var hmssForReachEnabled by Delegates.notNull() + private set + @CommandLine.Option( names = ["--direct-noise-mechanism"], description = diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt index 1d9f1fbc0ff..533df51e77d 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt @@ -102,13 +102,9 @@ class MeasurementsService( private val internalMeasurementsStub: InternalMeasurementsCoroutineStub, private val internalDataProvidersStub: InternalDataProvidersCoroutineStub, private val noiseMechanisms: List, - private val reachOnlyLlV2Enabled: Boolean, - /** - * Whether Honest Majority Share Shuffle (HMSS) is enabled. - * - * TODO(@renjiezh): Set this based on feature flag. - */ - private val hmssEnabled: Boolean = false, + private val reachOnlyLlV2Enabled: Boolean = false, + private val reachAndFrequencyHmssEnabled: Boolean = false, + private val reachOnlyHmssEnabled: Boolean = false, ) : MeasurementsCoroutineImplBase() { override suspend fun getMeasurement(request: GetMeasurementRequest): Measurement { @@ -496,7 +492,8 @@ class MeasurementsService( } } else { if ( - hmssEnabled && dataProviderCapabilities.all { it.honestMajorityShareShuffleSupported } + reachOnlyHmssEnabled && + dataProviderCapabilities.all { it.honestMajorityShareShuffleSupported } ) { protocolConfig { externalProtocolConfigId = HmssProtocolConfig.name @@ -535,7 +532,8 @@ class MeasurementsService( } } else { if ( - hmssEnabled && dataProviderCapabilities.all { it.honestMajorityShareShuffleSupported } + reachAndFrequencyHmssEnabled && + dataProviderCapabilities.all { it.honestMajorityShareShuffleSupported } ) { protocolConfig { externalProtocolConfigId = HmssProtocolConfig.name diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/ProtoConversions.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/ProtoConversions.kt index 0dae0dbf467..32d04e41f75 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/ProtoConversions.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/ProtoConversions.kt @@ -464,6 +464,7 @@ private fun buildMpcProtocolConfig( sketchParams = shareShuffleSketchParams { bytesPerRegister = protocolConfig.honestMajorityShareShuffle.sketchParams.bytesPerRegister + ringModulus = protocolConfig.honestMajorityShareShuffle.sketchParams.ringModulus } noiseMechanism = protocolConfig.honestMajorityShareShuffle.noiseMechanism.toNoiseMechanism() diff --git a/src/main/proto/wfa/measurement/internal/kingdom/protocol_config.proto b/src/main/proto/wfa/measurement/internal/kingdom/protocol_config.proto index 2721b345fd2..2163a0d3dc4 100644 --- a/src/main/proto/wfa/measurement/internal/kingdom/protocol_config.proto +++ b/src/main/proto/wfa/measurement/internal/kingdom/protocol_config.proto @@ -175,12 +175,9 @@ message LiquidLegionsSketchParams { // Parameters for a honest majority share shuffle sketch. message ShareShuffleSketchParams { - // The number of registers in the sketch. - int64 register_count = 1; - // Length of each register in bytes. - int32 bytes_per_register = 2; + int32 bytes_per_register = 1; // Secret share modulus. - uint32 ring_modulus = 3; + int32 ring_modulus = 2; } diff --git a/src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto b/src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto index cf0f7ecdadc..1e0e683aaad 100644 --- a/src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto +++ b/src/main/proto/wfa/measurement/internal/kingdom/protocol_config_config.proto @@ -39,6 +39,6 @@ message HmssProtocolConfigConfig { ProtocolConfig.HonestMajorityShareShuffle protocol_config = 1; // List of required duchies for this protocol. // - // The first one must correspond to the aggregator. + // HMSS protocol requires 3 duchies. repeated string required_external_duchy_ids = 2; } diff --git a/src/test/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsServiceTest.kt index 807609ea128..b5f6b444240 100644 --- a/src/test/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsServiceTest.kt @@ -241,7 +241,7 @@ class MeasurementsServiceTest { DataProvidersGrpcKt.DataProvidersCoroutineStub(grpcTestServerRule.channel), NOISE_MECHANISMS, reachOnlyLlV2Enabled = true, - hmssEnabled = true, + reachAndFrequencyHmssEnabled = true, ) } From 28184c48553ba11245c0533b80f5f016ec8b9bbc Mon Sep 17 00:00:00 2001 From: renjiez Date: Wed, 29 May 2024 05:04:39 +0000 Subject: [PATCH 02/12] Bug fix. --- .../mill/shareshuffle/HonestMajorityShareShuffleMill.kt | 2 +- .../mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle/HonestMajorityShareShuffleMill.kt b/src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle/HonestMajorityShareShuffleMill.kt index 5ca94b2c01c..60cb226bb98 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle/HonestMajorityShareShuffleMill.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle/HonestMajorityShareShuffleMill.kt @@ -128,7 +128,7 @@ class HonestMajorityShareShuffleMill( openTelemetry = openTelemetry, ) { init { - if (protocolSetupConfig.role == AGGREGATOR) { + if (protocolSetupConfig.role != AGGREGATOR) { requireNotNull(privateKeyStore) { "private key store is not set up." } } } diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt index b2da9538670..4bdeef5eed6 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt @@ -61,7 +61,7 @@ import org.wfanet.measurement.duchy.db.computation.ComputationDataClients import org.wfanet.measurement.duchy.storage.TinkKeyStore import org.wfanet.measurement.internal.duchy.ComputationStatsGrpcKt.ComputationStatsCoroutineStub import org.wfanet.measurement.internal.duchy.ComputationsGrpcKt.ComputationsCoroutineStub -import org.wfanet.measurement.internal.duchy.config.HonestMajorityShareShuffleSetupConfig +import org.wfanet.measurement.internal.duchy.config.ProtocolsSetupConfig import org.wfanet.measurement.storage.StorageClient import org.wfanet.measurement.system.v1alpha.ComputationControlGrpcKt.ComputationControlCoroutineStub import org.wfanet.measurement.system.v1alpha.ComputationLogEntriesGrpcKt.ComputationLogEntriesCoroutineStub as SystemComputationLogEntriesCoroutineStub @@ -231,7 +231,7 @@ abstract class HonestMajorityShareShuffleMillDaemon : Runnable { cryptoWorker = JniHonestMajorityShareShuffleCryptor(), protocolSetupConfig = flags.protocolsSetupConfig.reader().use { - parseTextProto(it, HonestMajorityShareShuffleSetupConfig.getDefaultInstance()) + parseTextProto(it, ProtocolsSetupConfig.getDefaultInstance()).honestMajorityShareShuffle }, workLockDuration = flags.workLockDuration, openTelemetry = openTelemetry, From 8269cd1c551f708a7d5d3577fc6122fe0878538b Mon Sep 17 00:00:00 2001 From: renjiez Date: Wed, 29 May 2024 05:46:38 +0000 Subject: [PATCH 03/12] Do no specify local mill resource --- src/main/k8s/local/duchies.cue | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/k8s/local/duchies.cue b/src/main/k8s/local/duchies.cue index b8eec545034..00d725fd4d9 100644 --- a/src/main/k8s/local/duchies.cue +++ b/src/main/k8s/local/duchies.cue @@ -124,14 +124,14 @@ duchies: [ cs_cert_resource_name: duchyConfig.certificateResourceName duchyKeyEncryptionKeyFile: duchyConfig.duchyKeyEncryptionKeyFile } - deployments: { - "honest-majority-share-shuffle-mill-daemon-deployment": { - _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements - } - } - } +// deployments: { +// "honest-majority-share-shuffle-mill-daemon-deployment": { +// _container: { +// _javaOptions: maxHeapSize: #MillMaxHeapSize +// resources: #MillResourceRequirements +// } +// } +// } } } if (duchyConfig.databaseType == "postgres") { @@ -155,12 +155,12 @@ duchies: [ _container: _envVars: EnvVars _updateSchemaContainer: _envVars: EnvVars } - "honest-majority-share-shuffle-mill-daemon-deployment": { - _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements - } - } +// "honest-majority-share-shuffle-mill-daemon-deployment": { +// _container: { +// _javaOptions: maxHeapSize: #MillMaxHeapSize +// resources: #MillResourceRequirements +// } +// } } } } From eb9e4e230d34925e1584d6dbeae997a4a0dd8f77 Mon Sep 17 00:00:00 2001 From: renjiez Date: Wed, 29 May 2024 17:42:07 +0000 Subject: [PATCH 04/12] Lint --- src/main/k8s/local/duchies.cue | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/main/k8s/local/duchies.cue b/src/main/k8s/local/duchies.cue index 00d725fd4d9..7e8282d6090 100644 --- a/src/main/k8s/local/duchies.cue +++ b/src/main/k8s/local/duchies.cue @@ -24,18 +24,6 @@ _worker2_cert_name: string @tag("worker2_cert_name") #KingdomPublicApiTarget: (#Target & {name: "v2alpha-public-api-server"}).target #SpannerEmulatorHost: (#Target & {name: "spanner-emulator"}).target -#MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { - requests: { - cpu: "3" - memory: "4Gi" - } - limits: { - memory: ResourceRequirements.requests.memory - } -} -#MillMaxHeapSize: "3500M" -#MillReplicas: 1 - #DuchyConfig: { let duchyName = name name: string @@ -124,14 +112,6 @@ duchies: [ cs_cert_resource_name: duchyConfig.certificateResourceName duchyKeyEncryptionKeyFile: duchyConfig.duchyKeyEncryptionKeyFile } -// deployments: { -// "honest-majority-share-shuffle-mill-daemon-deployment": { -// _container: { -// _javaOptions: maxHeapSize: #MillMaxHeapSize -// resources: #MillResourceRequirements -// } -// } -// } } } if (duchyConfig.databaseType == "postgres") { @@ -155,12 +135,6 @@ duchies: [ _container: _envVars: EnvVars _updateSchemaContainer: _envVars: EnvVars } -// "honest-majority-share-shuffle-mill-daemon-deployment": { -// _container: { -// _javaOptions: maxHeapSize: #MillMaxHeapSize -// resources: #MillResourceRequirements -// } -// } } } } From 6d0cde50c2018ff3ef60e0b62c8b3eadb45e7a18 Mon Sep 17 00:00:00 2001 From: renjiez Date: Wed, 5 Jun 2024 21:50:15 +0000 Subject: [PATCH 05/12] Update based on PR comments. --- .../duchy/deploy/common/daemon/herald/HeraldDaemon.kt | 2 ++ .../mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt | 2 +- .../kingdom/service/api/v2alpha/MeasurementsService.kt | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt index 4e26c97ce78..b1b4e361b23 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/herald/HeraldDaemon.kt @@ -88,6 +88,7 @@ class HeraldFlags { description = ["The key encryption key file (binary format) used for private key store."], ) var keyEncryptionKeyTinkFile: File? = null + private set @CommandLine.Option( names = ["--deletable-computation-state"], @@ -154,6 +155,7 @@ abstract class HeraldDaemon : Runnable { // This will be the name of the pod when deployed to Kubernetes. val heraldId = System.getenv("HOSTNAME") + // TODO(@renjiez): Read from a KMS-encrypted store instead. val privateKeyStore = flags.keyEncryptionKeyTinkFile?.let { file -> val keyUri = FakeKmsClient.KEY_URI_PREFIX + "kek" diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt index 4bdeef5eed6..0c3f7e2cb08 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt @@ -137,7 +137,7 @@ abstract class HonestMajorityShareShuffleMillDaemon : Runnable { val csX509Certificate = flags.csCertificateDerFile.inputStream().use { input -> readCertificate(input) } val csCertificate = Certificate(flags.csCertificateName, csX509Certificate) - // TODO: Read from a KMS-encrypted store instead. + // TODO(@renjiez): Read from a KMS-encrypted store instead. val csSigningKey = SigningKeyHandle( csX509Certificate, diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt index 533df51e77d..e198d81bde2 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/api/v2alpha/MeasurementsService.kt @@ -103,6 +103,7 @@ class MeasurementsService( private val internalDataProvidersStub: InternalDataProvidersCoroutineStub, private val noiseMechanisms: List, private val reachOnlyLlV2Enabled: Boolean = false, + // TODO(@renjiez): merge the two options below once implementing reach-only HMSS. private val reachAndFrequencyHmssEnabled: Boolean = false, private val reachOnlyHmssEnabled: Boolean = false, ) : MeasurementsCoroutineImplBase() { From fc3b068ddc30a7b85cd6b161095ab51d869e562d Mon Sep 17 00:00:00 2001 From: renjiez Date: Thu, 6 Jun 2024 23:56:34 +0000 Subject: [PATCH 06/12] Update based on PR comment. --- src/main/k8s/dev/duchy_eks.cue | 35 ++++++++++++++++++++++------------ src/main/k8s/dev/duchy_gke.cue | 31 ++++++++++++++++++++---------- src/main/k8s/duchy.cue | 18 ++++++++--------- 3 files changed, 53 insertions(+), 31 deletions(-) diff --git a/src/main/k8s/dev/duchy_eks.cue b/src/main/k8s/dev/duchy_eks.cue index f2a941d27de..c7529240904 100644 --- a/src/main/k8s/dev/duchy_eks.cue +++ b/src/main/k8s/dev/duchy_eks.cue @@ -39,24 +39,35 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)" #HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "25m" - memory: "1024Mi" + memory: "512M" } limits: { memory: ResourceRequirements.requests.memory } } -#HeraldMaxHeapSize: "800M" -#MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { +#HeraldMaxHeapSize: "400M" +#Llv2MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "3" - memory: "4Gi" + memory: "2.5Gi" } limits: { memory: ResourceRequirements.requests.memory } } -#MillMaxHeapSize: "3500M" -#MillReplicas: 1 +#Llv2MillMaxHeapSize: "1G" +#Llv2MillReplicas: 1 +#HmssMillResourceRequirements: ResourceRequirements=#ResourceRequirements & { + requests: { + cpu: "2" + memory: "6Gi" + } + limits: { + memory: ResourceRequirements.requests.memory + } +} +#HmssMillMaxHeapSize: "5G" +#Llv2MillReplicas: 1 #FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "200m" @@ -120,11 +131,11 @@ duchy: #PostgresDuchy & { "liquid-legions-v2-mill-daemon-deployment": { _workLockDuration: "10m" _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements + _javaOptions: maxHeapSize: #Llv2MillMaxHeapSize + resources: #Llv2MillResourceRequirements } spec: { - replicas: #MillReplicas + replicas: #Llv2MillReplicas template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { serviceAccountName: #StorageServiceAccount } @@ -133,11 +144,11 @@ duchy: #PostgresDuchy & { "honest-majority-share-shuffle-mill-daemon-deployment": { _workLockDuration: "5m" _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements + _javaOptions: maxHeapSize: #HmssMillMaxHeapSize + resources: #HmssMillResourceRequirements } spec: { - replicas: #MillReplicas + replicas: #HmssMillReplicas template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { serviceAccountName: #StorageServiceAccount } diff --git a/src/main/k8s/dev/duchy_gke.cue b/src/main/k8s/dev/duchy_gke.cue index 96ea64204ea..8a6d1c6a615 100644 --- a/src/main/k8s/dev/duchy_gke.cue +++ b/src/main/k8s/dev/duchy_gke.cue @@ -46,18 +46,29 @@ _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)" memory: ResourceRequirements.requests.memory } } -#HeraldMaxHeapSize: "400M" -#MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { +#HeraldMaxHeapSize: "400M" +#Llv2MillResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "3" + memory: "2.5Gi" + } + limits: { + memory: ResourceRequirements.requests.memory + } +} +#Llv2MillMaxHeapSize: "1G" +#Llv2MillReplicas: 1 +#HmssMillResourceRequirements: ResourceRequirements=#ResourceRequirements & { + requests: { + cpu: "2" memory: "6Gi" } limits: { memory: ResourceRequirements.requests.memory } } -#MillMaxHeapSize: "5G" -#MillReplicas: 1 +#HmssMillMaxHeapSize: "5G" +#HmssMillReplicas: 1 #FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "200m" @@ -137,11 +148,11 @@ duchy: #SpannerDuchy & { "liquid-legions-v2-mill-daemon-deployment": { _workLockDuration: "10m" _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements + _javaOptions: maxHeapSize: #Llv2MillMaxHeapSize + resources: #Llv2MillResourceRequirements } spec: { - replicas: #MillReplicas + replicas: #Llv2MillReplicas template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { serviceAccountName: #StorageServiceAccount } @@ -150,11 +161,11 @@ duchy: #SpannerDuchy & { "honest-majority-share-shuffle-mill-daemon-deployment": { _workLockDuration: "5m" _container: { - _javaOptions: maxHeapSize: #MillMaxHeapSize - resources: #MillResourceRequirements + _javaOptions: maxHeapSize: #HmssMillMaxHeapSize + resources: #HmssMillResourceRequirements } spec: { - replicas: #MillReplicas + replicas: #HmssMillReplicas template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & { serviceAccountName: #StorageServiceAccount } diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index 5807393850e..77547dcb41b 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -20,10 +20,10 @@ import ("strings") #Duchy: { _duchy: { - name: string - protocols_setup_config: string - cs_cert_resource_name: string - duchyKeyEncryptionKeyFile: string + name: string + protocols_setup_config: string + cs_cert_resource_name: string + duchyKeyEncryptionKeyFile?: string } _duchy_secret_name: string _computation_control_targets: [Name=_]: string @@ -35,10 +35,10 @@ import ("strings") _blob_storage_flags: [...string] _verbose_grpc_logging: "true" | "false" - _name: _duchy.name - _protocols_setup_config: _duchy.protocols_setup_config - _cs_cert_resource_name: _duchy.cs_cert_resource_name - _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile + _name: _duchy.name + _protocols_setup_config: _duchy.protocols_setup_config + _cs_cert_resource_name: _duchy.cs_cert_resource_name + _duchyKeyEncryptionKeyFile?: _duchy.duchyKeyEncryptionKeyFile _object_prefix: "\(_name)-" @@ -76,7 +76,7 @@ import ("strings") _duchy_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/\(_name)_tls.pem" _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" - _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" + _duchyKeyEncryptionKeyFileFlag?: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" From ac128a09a956f22327932efc5a8c242ffc7c0969 Mon Sep 17 00:00:00 2001 From: renjiez Date: Fri, 7 Jun 2024 00:25:56 +0000 Subject: [PATCH 07/12] Update for opentelemetry --- src/main/k8s/dev/duchy_eks.cue | 2 +- src/main/k8s/duchy.cue | 4 +- .../daemon/mill/shareshuffle/BUILD.bazel | 5 -- .../HonestMajorityShareShuffleMillDaemon.kt | 62 +------------------ .../HonestMajorityShareShuffleMillFlags.kt | 20 ------ src/main/terraform/aws/cmms/duchies.tf | 2 +- src/main/terraform/gcloud/cmms/duchies.tf | 2 +- 7 files changed, 7 insertions(+), 90 deletions(-) diff --git a/src/main/k8s/dev/duchy_eks.cue b/src/main/k8s/dev/duchy_eks.cue index c7529240904..2abc51e2e02 100644 --- a/src/main/k8s/dev/duchy_eks.cue +++ b/src/main/k8s/dev/duchy_eks.cue @@ -67,7 +67,7 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)" } } #HmssMillMaxHeapSize: "5G" -#Llv2MillReplicas: 1 +#HmssMillReplicas: 1 #FulfillmentResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "200m" diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index ea75a5f5c6e..63b4f0df997 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -159,7 +159,7 @@ import ("strings") "\(_name)-internal-api-server", "\(_name)-computation-control-server", ] } - "honest-majority-share-shuffle-mill-daemon-deployment": Deployment={ + "honest-majority-share-shuffle-mill-daemon-deployment": { _workLockDuration?: string _container: args: [ _duchyInternalApiTargetFlag, @@ -180,8 +180,6 @@ import ("strings") if (_duchyKeyEncryptionKeyFile != _|_) {_duchyKeyEncryptionKeyFileFlag}, if (_millPollingInterval != _|_) {"--polling-interval=\(_millPollingInterval)"}, if (_workLockDuration != _|_) {"--work-lock-duration=\(_workLockDuration)"}, - _otlpEndpoint, - "--otel-service-name=\(Deployment.metadata.name)", ] + _blob_storage_flags + _computation_control_target_flags spec: template: spec: _dependencies: [ "\(_name)-internal-api-server", "\(_name)-computation-control-server", diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel index cb0daa2d312..e1ed7176989 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/BUILD.bazel @@ -13,11 +13,6 @@ kt_jvm_library( ], visibility = ["//src/main/kotlin/org/wfanet/measurement/duchy/deploy:__subpackages__"], deps = [ - "//imports/java/io/opentelemetry/exporter/otlp", - "//imports/java/io/opentelemetry/sdk", - "//imports/java/io/opentelemetry/sdk:common", - "//imports/java/io/opentelemetry/sdk:metrics", - "//imports/java/io/opentelemetry/semconv", "//src/main/kotlin/org/wfanet/measurement/common/identity", "//src/main/kotlin/org/wfanet/measurement/duchy/daemon/mill/shareshuffle:honest_majority_share_shuffle_mill", "//src/main/kotlin/org/wfanet/measurement/duchy/db/computation", diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt index 0c3f7e2cb08..695af39f2a3 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt @@ -23,19 +23,7 @@ import com.google.protobuf.ByteString import io.grpc.Channel import io.opentelemetry.api.GlobalOpenTelemetry import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.common.Attributes -import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter -import io.opentelemetry.sdk.OpenTelemetrySdk -import io.opentelemetry.sdk.metrics.Aggregation -import io.opentelemetry.sdk.metrics.InstrumentSelector -import io.opentelemetry.sdk.metrics.InstrumentType -import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.View -import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader -import io.opentelemetry.sdk.resources.Resource -import io.opentelemetry.semconv.ResourceAttributes import java.time.Clock -import java.time.Duration import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext @@ -165,53 +153,9 @@ abstract class HonestMajorityShareShuffleMillDaemon : Runnable { .makeKmsPrivateKeyStore(TinkKeyStore(storageClient), keyUri) } - val openTelemetry: OpenTelemetry = - if (flags.openTelemetryOptions == null) { - GlobalOpenTelemetry.get() - } else { - val endpoint = flags.openTelemetryOptions!!.otelExporterOtlpEndpoint - val serviceName = flags.openTelemetryOptions!!.otelServiceName - val resource: Resource = - Resource.getDefault() - .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, serviceName))) - val meterProvider = - SdkMeterProvider.builder() - .setResource(resource) - .registerMetricReader( - PeriodicMetricReader.builder( - OtlpGrpcMetricExporter.builder() - .setTimeout(Duration.ofSeconds(30L)) - .setEndpoint(endpoint) - .build() - ) - .setInterval(Duration.ofSeconds(60L)) - .build() - ) - .registerView( - InstrumentSelector.builder().setType(InstrumentType.HISTOGRAM).build(), - View.builder() - .setAggregation( - Aggregation.explicitBucketHistogram( - listOf( - 1000.0, - 2000.0, - 4000.0, - 8000.0, - 16000.0, - 32000.0, - 64000.0, - 128000.0, - 256000.0, - 512000.0, - 1024000.0, - ) - ) - ) - .build(), - ) - .build() - OpenTelemetrySdk.builder().setMeterProvider(meterProvider).build() - } + // OpenTelemetry is usually enabled using the Java agent, in which case this will grab the + // shared instance. + val openTelemetry: OpenTelemetry = GlobalOpenTelemetry.get() val mill = HonestMajorityShareShuffleMill( diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt index a6bcbf0a14d..3039114b06d 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt @@ -127,24 +127,4 @@ class HonestMajorityShareShuffleMillFlags { description = ["The key encryption key file (binary format) used for private key store."], ) var keyEncryptionKeyTinkFile: File? = null - - @CommandLine.ArgGroup(exclusive = false) var openTelemetryOptions: OpenTelemetryOptions? = null - // All options here must be present or none of them must be present. - class OpenTelemetryOptions { - @CommandLine.Option( - names = ["--otel-exporter-otlp-endpoint"], - description = ["Endpoint for OpenTelemetry Collector."], - required = true, - ) - lateinit var otelExporterOtlpEndpoint: String - private set - - @CommandLine.Option( - names = ["--otel-service-name"], - description = ["Service name to label duchy metrics with."], - required = true, - ) - lateinit var otelServiceName: String - private set - } } diff --git a/src/main/terraform/aws/cmms/duchies.tf b/src/main/terraform/aws/cmms/duchies.tf index c733134e09a..be924122cd0 100644 --- a/src/main/terraform/aws/cmms/duchies.tf +++ b/src/main/terraform/aws/cmms/duchies.tf @@ -30,7 +30,7 @@ module "clusters" { default_instance_types = ["m5.large"] default_max_node_count = 2 high_perf_instance_types = ["c5.xlarge"] - high_perf_max_node_count = 2 + high_perf_max_node_count = 3 } # IAM role to be used by aws_load_balancer_controller addons across all clusters diff --git a/src/main/terraform/gcloud/cmms/duchies.tf b/src/main/terraform/gcloud/cmms/duchies.tf index 92867de9a93..1080cd40505 100644 --- a/src/main/terraform/gcloud/cmms/duchies.tf +++ b/src/main/terraform/gcloud/cmms/duchies.tf @@ -41,7 +41,7 @@ module "highmem_node_pools" { name = "highmem" service_account = module.common.cluster_service_account machine_type = "c2-standard-4" - max_node_count = 2 + max_node_count = 3 spot = true } From 139543ab1d290e67061d807178bd10636a28fee9 Mon Sep 17 00:00:00 2001 From: renjiez Date: Fri, 7 Jun 2024 16:38:30 +0000 Subject: [PATCH 08/12] Revert optional sign in cue --- ..._aggregator_secret_kustomization.tmpl.yaml | 2 +- src/main/k8s/duchy.cue | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml b/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml index d1e33e75e29..5505923d04c 100644 --- a/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml +++ b/src/main/k8s/dev/non_aggregator_secret_kustomization.tmpl.yaml @@ -21,4 +21,4 @@ secretGenerator: - {duchy_id}_cs_private.der - duchy_cert_config.textproto - {duchy_id}_protocols_setup_config.textproto - - {duchy_id}_kek.tink \ No newline at end of file + - {duchy_id}_kek.tink diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index 63b4f0df997..a946a40cf09 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -20,10 +20,10 @@ import ("strings") #Duchy: { _duchy: { - name: string - protocols_setup_config: string - cs_cert_resource_name: string - duchyKeyEncryptionKeyFile?: string + name: string + protocols_setup_config: string + cs_cert_resource_name: string + duchyKeyEncryptionKeyFile: string } _duchy_secret_name: string _computation_control_targets: [Name=_]: string @@ -35,10 +35,10 @@ import ("strings") _blob_storage_flags: [...string] _verbose_grpc_logging: "true" | "false" - _name: _duchy.name - _protocols_setup_config: _duchy.protocols_setup_config - _cs_cert_resource_name: _duchy.cs_cert_resource_name - _duchyKeyEncryptionKeyFile?: _duchy.duchyKeyEncryptionKeyFile + _name: _duchy.name + _protocols_setup_config: _duchy.protocols_setup_config + _cs_cert_resource_name: _duchy.cs_cert_resource_name + _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile _object_prefix: "\(_name)-" @@ -64,7 +64,7 @@ import ("strings") } } - _millPollingInterval?: string + _millPollingInterval: string _duchyInternalServerContainerArgs: [...string] _akid_to_principal_map_file_flag: "--authority-key-identifier-to-principal-map-file=/etc/\(#AppName)/config-files/authority_key_identifier_to_principal_map.textproto" @@ -76,7 +76,7 @@ import ("strings") _duchy_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/\(_name)_tls.pem" _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" - _duchyKeyEncryptionKeyFileFlag?: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" + _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" From 25769c49f4f725bbafed011c69165ebc83aa5479 Mon Sep 17 00:00:00 2001 From: renjiez Date: Fri, 7 Jun 2024 17:16:19 +0000 Subject: [PATCH 09/12] update optional duchyKeyEncryptionKeyFile --- src/main/k8s/duchy.cue | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index a946a40cf09..6324c6f3f82 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -20,10 +20,10 @@ import ("strings") #Duchy: { _duchy: { - name: string - protocols_setup_config: string - cs_cert_resource_name: string - duchyKeyEncryptionKeyFile: string + name: string + protocols_setup_config: string + cs_cert_resource_name: string + duchyKeyEncryptionKeyFile?: string } _duchy_secret_name: string _computation_control_targets: [Name=_]: string @@ -35,10 +35,13 @@ import ("strings") _blob_storage_flags: [...string] _verbose_grpc_logging: "true" | "false" - _name: _duchy.name - _protocols_setup_config: _duchy.protocols_setup_config - _cs_cert_resource_name: _duchy.cs_cert_resource_name - _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile + _name: _duchy.name + _protocols_setup_config: _duchy.protocols_setup_config + _cs_cert_resource_name: _duchy.cs_cert_resource_name + _duchyKeyEncryptionKeyFile?: string + if (_duchy.duchyKeyEncryptionKeyFile != _|_) { + _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile + } _object_prefix: "\(_name)-" @@ -76,15 +79,18 @@ import ("strings") _duchy_tls_cert_file_flag: "--tls-cert-file=/var/run/secrets/files/\(_name)_tls.pem" _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" - _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" - _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target - _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" - _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" - _duchyDryRunRetentionPolicyFlag: "--dry-run" - _duchyMillParallelismFlag: "--parallelism=\(_duchyMillParallelism)" - _duchy_cs_cert_file_flag: "--consent-signaling-certificate-der-file=/var/run/secrets/files/\(_name)_cs_cert.der" - _duchy_cs_key_file_flag: "--consent-signaling-private-key-der-file=/var/run/secrets/files/\(_name)_cs_private.der" - _duchy_cs_cert_rename_name_flag: "--consent-signaling-certificate-resource-name=\(_cs_cert_resource_name)" + _duchyKeyEncryptionKeyFileFlag?: string + if (_duchyKeyEncryptionKeyFile != _|_) { + _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" + } + _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target + _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" + _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" + _duchyDryRunRetentionPolicyFlag: "--dry-run" + _duchyMillParallelismFlag: "--parallelism=\(_duchyMillParallelism)" + _duchy_cs_cert_file_flag: "--consent-signaling-certificate-der-file=/var/run/secrets/files/\(_name)_cs_cert.der" + _duchy_cs_key_file_flag: "--consent-signaling-private-key-der-file=/var/run/secrets/files/\(_name)_cs_private.der" + _duchy_cs_cert_rename_name_flag: "--consent-signaling-certificate-resource-name=\(_cs_cert_resource_name)" _duchyDeletableStatesFlag: [ for state in _deletableComputationStates {"--deletable-computation-state=\(state)"}] _kingdom_system_api_target_flag: "--kingdom-system-api-target=\(_kingdom_system_api_target)" _kingdom_system_api_cert_host_flag: "--kingdom-system-api-cert-host=localhost" From 7ecb073ab0b53d6054b21c7ea7d338daeec971e6 Mon Sep 17 00:00:00 2001 From: renjiez Date: Fri, 7 Jun 2024 20:38:09 +0000 Subject: [PATCH 10/12] Update docs --- docs/eks/duchy-deployment.md | 4 ++++ docs/gke/duchy-deployment.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/eks/duchy-deployment.md b/docs/eks/duchy-deployment.md index 216dfb407ff..05cae339a0f 100644 --- a/docs/eks/duchy-deployment.md +++ b/docs/eks/duchy-deployment.md @@ -217,6 +217,10 @@ files are required in a Duchy: - Set the role (aggregator or non_aggregator) in the config appropriately - [Example](../../src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto) +1. `worker2_kek.tink` + Key encryption key used in HMSS protocol to encrypt tink key pairs. + - [Example](../../src/main/k8s/testing/secretfiles/worker2_kek.tink) + Place these files into the `src/main/k8s/dev/worker2_duchy_secret/` path within the Kustomization directory. diff --git a/docs/gke/duchy-deployment.md b/docs/gke/duchy-deployment.md index 8e36072b6d3..f77dbb32a01 100644 --- a/docs/gke/duchy-deployment.md +++ b/docs/gke/duchy-deployment.md @@ -213,6 +213,10 @@ files are required in a Duchy: - Set the role (aggregator or non_aggregator) in the config appropriately - [Example](../../src/main/k8s/testing/secretfiles/aggregator_protocols_setup_config.textproto) +1. `worker2_kek.tink` + Key encryption key used in HMSS protocol to encrypt tink key pairs. + - [Example](../../src/main/k8s/testing/secretfiles/worker2_kek.tink) + Place these files into the `src/main/k8s/dev/worker1_duchy_secret/` path within the Kustomization directory. From 83428bd8214b76c3795ef9e06f19b88096d3edb8 Mon Sep 17 00:00:00 2001 From: renjiez Date: Fri, 7 Jun 2024 22:07:52 +0000 Subject: [PATCH 11/12] Update based on PR comments. #2 --- src/main/k8s/dev/duchy_eks.cue | 2 +- src/main/k8s/dev/duchy_gke.cue | 2 +- src/main/k8s/duchy.cue | 24 ++++++++----------- .../HonestMajorityShareShuffleMillFlags.kt | 1 + 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/k8s/dev/duchy_eks.cue b/src/main/k8s/dev/duchy_eks.cue index 2abc51e2e02..e32ee27ad27 100644 --- a/src/main/k8s/dev/duchy_eks.cue +++ b/src/main/k8s/dev/duchy_eks.cue @@ -39,7 +39,7 @@ _duchyCertName: "duchies/\(_duchyName)/certificates/\(_certificateId)" #HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "25m" - memory: "512M" + memory: "512Mi" } limits: { memory: ResourceRequirements.requests.memory diff --git a/src/main/k8s/dev/duchy_gke.cue b/src/main/k8s/dev/duchy_gke.cue index 8a6d1c6a615..90f379f9208 100644 --- a/src/main/k8s/dev/duchy_gke.cue +++ b/src/main/k8s/dev/duchy_gke.cue @@ -40,7 +40,7 @@ _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)" #HeraldResourceRequirements: ResourceRequirements=#ResourceRequirements & { requests: { cpu: "25m" - memory: "512M" + memory: "512Mi" } limits: { memory: ResourceRequirements.requests.memory diff --git a/src/main/k8s/duchy.cue b/src/main/k8s/duchy.cue index 6324c6f3f82..8757f572d8a 100644 --- a/src/main/k8s/duchy.cue +++ b/src/main/k8s/duchy.cue @@ -39,9 +39,7 @@ import ("strings") _protocols_setup_config: _duchy.protocols_setup_config _cs_cert_resource_name: _duchy.cs_cert_resource_name _duchyKeyEncryptionKeyFile?: string - if (_duchy.duchyKeyEncryptionKeyFile != _|_) { - _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile - } + _duchyKeyEncryptionKeyFile: _duchy.duchyKeyEncryptionKeyFile _object_prefix: "\(_name)-" @@ -80,17 +78,15 @@ import ("strings") _duchy_tls_key_file_flag: "--tls-key-file=/var/run/secrets/files/\(_name)_tls.key" _duchy_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" _duchyKeyEncryptionKeyFileFlag?: string - if (_duchyKeyEncryptionKeyFile != _|_) { - _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" - } - _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target - _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" - _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" - _duchyDryRunRetentionPolicyFlag: "--dry-run" - _duchyMillParallelismFlag: "--parallelism=\(_duchyMillParallelism)" - _duchy_cs_cert_file_flag: "--consent-signaling-certificate-der-file=/var/run/secrets/files/\(_name)_cs_cert.der" - _duchy_cs_key_file_flag: "--consent-signaling-private-key-der-file=/var/run/secrets/files/\(_name)_cs_private.der" - _duchy_cs_cert_rename_name_flag: "--consent-signaling-certificate-resource-name=\(_cs_cert_resource_name)" + _duchyKeyEncryptionKeyFileFlag: "--key-encryption-key-file=/var/run/secrets/files/\(_duchyKeyEncryptionKeyFile)" + _duchyInternalApiTargetFlag: "--computations-service-target=" + (#Target & {name: "\(_name)-internal-api-server"}).target + _duchyInternalApiCertHostFlag: "--computations-service-cert-host=localhost" + _duchyComputationsTimeToLiveFlag: "--computations-time-to-live=\(_computationsTimeToLive)" + _duchyDryRunRetentionPolicyFlag: "--dry-run" + _duchyMillParallelismFlag: "--parallelism=\(_duchyMillParallelism)" + _duchy_cs_cert_file_flag: "--consent-signaling-certificate-der-file=/var/run/secrets/files/\(_name)_cs_cert.der" + _duchy_cs_key_file_flag: "--consent-signaling-private-key-der-file=/var/run/secrets/files/\(_name)_cs_private.der" + _duchy_cs_cert_rename_name_flag: "--consent-signaling-certificate-resource-name=\(_cs_cert_resource_name)" _duchyDeletableStatesFlag: [ for state in _deletableComputationStates {"--deletable-computation-state=\(state)"}] _kingdom_system_api_target_flag: "--kingdom-system-api-target=\(_kingdom_system_api_target)" _kingdom_system_api_cert_host_flag: "--kingdom-system-api-cert-host=localhost" diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt index 3039114b06d..13f1d016c92 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillFlags.kt @@ -26,6 +26,7 @@ import org.wfanet.measurement.duchy.deploy.common.SystemApiFlags import picocli.CommandLine class HonestMajorityShareShuffleMillFlags { + // TODO(@renjie): Extract common mill flags. @CommandLine.Mixin lateinit var duchy: CommonDuchyFlags private set From d08fb969d805f2bd838e7578abdfc949f20858c0 Mon Sep 17 00:00:00 2001 From: renjiez Date: Tue, 11 Jun 2024 21:36:57 +0000 Subject: [PATCH 12/12] Add todo for millId --- .../mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt index 695af39f2a3..beb5b93d84d 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/daemon/mill/shareshuffle/HonestMajorityShareShuffleMillDaemon.kt @@ -136,6 +136,7 @@ abstract class HonestMajorityShareShuffleMillDaemon : Runnable { // This will be the name of the pod when deployed to Kubernetes. Note that the millId is // included in mill logs to help debugging. + // TODO(@renjiez): add the parameter to override the millId. val millId = System.getenv("HOSTNAME") val privateKeyStore =