Release 4.5.0 (2024-12-03)

### Features * [Private Aggregation] Make ig index available to populate interest_group_index in PrivateAggrgeteContribution * [Private Aggregation] Populate id_idx for PrivateAggregateContribution * Add allow_compressed_auction_config flag to Terraform * Add bidding.business_logic.debug_url_count metric * Add bidding.business_logic.debug_urls_size_bytes metric * Add bidding.business_logic.failed_to_bid_percent metric * Add compressed auction config field to API * Add dimension for GenerateBids timed out errors to bidding.errors_count * Add TCMalloc config for B&A inference * Allow model eviction grace period * Control whether scoring signals are required for scoring via flag * Enable TCMalloc for the TensorFlow sidecar * Read compressed AuctionConfig in SFE * remove metric noise for sampled debug request * Report model eviction via metrics * With Flag Allow Just Component Signals ### Bug Fixes * Allocate compression output buffer on heap * Missing inference metric label in AWS dashboard * Set default value for PRESUBMIT_PERF to 0 * Standardize business logic metrics for generateBids PA and PAS * Update tag to correctly pass roma_timeout_ms value to Roma ### Documentation * [BYOB] Add guide and sample request for generateBid SDK Bug: N/A Change-Id: I79c8e369bb814a5bc49919fc246e3739276a2cc9 GitOrigin-RevId: 4451bd75de7735f0428b917bbda4fea5b2f14e44
privacysandbox · Dec 4, 2024 · 6d1b383 · 6d1b383
1 parent 8b1a980
commit 6d1b383
Show file tree

Hide file tree

Showing 143 changed files with 5,112 additions and 1,238 deletions.
diff --git a/.markdown-link-check.json b/.markdown-link-check.json
@@ -5,6 +5,18 @@
     },
     {
       "pattern": "^http://go/"
+    },
+    {
+      "pattern": "^/tools/"
+    },
+    {
+      "pattern": "specs/udf_interface.proto"
+    },
+    {
+      "pattern": "udf/Communication%20Interface.md"
+    },
+    {
+      "pattern": "udf/Execution%20Environment%20and%20Interface.md"
     }
   ],
   "replacementPatterns": [

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,42 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 4.5.0 (2024-12-03)
+
+
+### Features
+
+* [Private Aggregation] Make ig index available to populate interest_group_index in PrivateAggrgeteContribution
+* [Private Aggregation] Populate id_idx for PrivateAggregateContribution
+* Add allow_compressed_auction_config flag to Terraform
+* Add bidding.business_logic.debug_url_count metric
+* Add bidding.business_logic.debug_urls_size_bytes metric
+* Add bidding.business_logic.failed_to_bid_percent metric
+* Add compressed auction config field to API
+* Add dimension for GenerateBids timed out errors to bidding.errors_count
+* Add TCMalloc config for B&A inference
+* Allow model eviction grace period
+* Control whether scoring signals are required for scoring via flag
+* Enable TCMalloc for the TensorFlow sidecar
+* Read compressed AuctionConfig in SFE
+* remove metric noise for sampled debug request
+* Report model eviction via metrics
+* With Flag Allow Just Component Signals
+
+
+### Bug Fixes
+
+* Allocate compression output buffer on heap
+* Missing inference metric label in AWS dashboard
+* Set default value for PRESUBMIT_PERF to 0
+* Standardize business logic metrics for generateBids PA and PAS
+* Update tag to correctly pass roma_timeout_ms value to Roma
+
+
+### Documentation
+
+* [BYOB] Add guide and sample request for generateBid SDK
+
 ## 4.4.0 (2024-11-14)
 
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -15,11 +15,11 @@ python_register_toolchains("//builders/bazel")
 
 http_archive(
     name = "google_privacysandbox_servers_common",
-    # 2024-10-09
-    sha256 = "7a0337420161304c7429c727b1f82394bc27e1e2586d2da30e6d6100ba92b437",
-    strip_prefix = "data-plane-shared-libraries-158593616a63df924af1cb689f3915b8d32e9db1",
+    # 2024-11-15
+    sha256 = "ed6b6913c16a5948cf75519d37aa35805ba7b73f0333f6968e534fa9f08db3fd",
+    strip_prefix = "data-plane-shared-libraries-96f555a9c901a31c03c426fddc128a77973535db",
     urls = [
-        "https://github.com/privacysandbox/data-plane-shared-libraries/archive/158593616a63df924af1cb689f3915b8d32e9db1.zip",
+        "https://github.com/privacysandbox/data-plane-shared-libraries/archive/96f555a9c901a31c03c426fddc128a77973535db.zip",
     ],
 )
 

diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto
@@ -21,6 +21,12 @@ import "google/api/annotations.proto";
 import "google/protobuf/struct.proto";
 import "src/logger/logger.proto";
 
+// Supported compression algorithms.
+enum CompressionAlgorithm {
+  COMPRESSION_ALGORITHM_UNKNOWN = 0;
+  COMPRESSION_ALGORITHM_GZIP = 1;
+}
+
 // ProtectedAudienceInput is generated and encrypted by the client,
 // passed through the untrusted Seller service, and decrypted by the
 // SellerFrontEnd service.
@@ -414,7 +420,7 @@ message AuctionResult {
   // BuyerReportingId of the winning Ad.
   // This will be verified with the buyerReportingId in the Ad properties on
   // the browser.
-  string buyer_reporting_id = 19;
+  optional string buyer_reporting_id = 19;
 
   // BuyerAndSellerReportingId of the winning Ad.
   // This will be verified with the buyerAndSellerReportingId in the Ad
@@ -562,7 +568,7 @@ message AuctionResult {
       // BuyerReportingId of the ghost winning Ad.
       // This will be verified with the buyerAndSellerReportingId in the Ad
       // properties on the browser.
-      string buyer_reporting_id = 6;
+      optional string buyer_reporting_id = 6;
 
       // SelectableBuyerAndSellerReportingId of the ghost winning Ad.
       // This will be verified with the buyerAndSellerReportingId in the Ad
@@ -868,6 +874,7 @@ message SelectAdRequest {
     // 'PerBuyerConfig.priority_signals_overrides'.
     string priority_signals = 13;
   }
+
   message ComponentAuctionResult {
     //  AuctionResult from a server component auction.
     bytes auction_result_ciphertext = 1;
@@ -882,7 +889,7 @@ message SelectAdRequest {
   bytes protected_audience_ciphertext = 1 [deprecated = true];
 
   // Plaintext. Passed by the untrusted Seller service.
-  AuctionConfig auction_config = 2;
+  AuctionConfig auction_config = 2 [deprecated = true];
 
   // Type of end user's device / client, that would help in validating the
   // client integrity. Also passed to the key/value services.
@@ -895,6 +902,19 @@ message SelectAdRequest {
   // Encrypted Component Auction Results for conducting a top level auction.
   // This should be populated along with protected_auction_ciphertext.
   repeated ComponentAuctionResult component_auction_results = 5;
+
+  message CompressedAuctionConfig {
+    // Compression algorithm used to compress the AuctionConfig.
+    CompressionAlgorithm compression_algorithm = 1;
+
+    // Compressed AuctionConfig proto bytestring.
+    bytes auction_config = 2;
+  }
+
+  // The `compressed_auction_config` field provides a compressed version of the
+  // AuctionConfig proto that adtechs can provide in place of the 'auction_config'
+  // field to reduce network costs.
+  CompressedAuctionConfig compressed_auction_config = 6;
 }
 
 // SelectAdResponse is sent from the SellerFrontEndService to the Seller
@@ -1133,7 +1153,7 @@ message AdWithBid {
   string bid_currency = 10;
 
   // Optional. buyerReportingId set in buyerReportingMetadata of reportWin()
-  string buyer_reporting_id = 11;
+  optional string buyer_reporting_id = 11;
 
   // Private aggregation object.
   repeated PrivateAggregateContribution private_aggregation_contributions = 12;
@@ -1144,7 +1164,7 @@ message AdWithBid {
 
   // Optional. selectedBuyerAndSellerReportingId in buyerReportingMetadata of
   // reportWin(). If provided, this will be substituted for interestGroupName.
-  string selected_buyer_and_seller_reporting_id = 14;
+  optional string selected_buyer_and_seller_reporting_id = 14;
 
   // Optional. Value of the data version header from the buyer kv server.
   // This is passed through to the buyer reporting function.
@@ -1556,7 +1576,7 @@ message ScoreAdsRequest {
       string interest_group_origin = 13;
 
       // Optional. buyerReportingId set in buyerReportingMetadata of reportWin()
-      string buyer_reporting_id = 14;
+      optional string buyer_reporting_id = 14;
 
       // Optional. buyerAndSellerReportingId in buyerReportingMetadata
       // of reportWin(). If provided, and selectedBuyerAndSellerReportingId
@@ -1576,6 +1596,10 @@ message ScoreAdsRequest {
       // Optional. Value of the data version header from the buyer kv server.
       // This is passed through to the buyer reporting function.
       uint32 data_version = 19;
+
+      // Interest group index corresponding to this Ad.
+      // This is required to populate the PrivateAggregateContribution.
+      int32 interest_group_idx = 20;
     }
 
     // Ad with bid.
@@ -1861,7 +1885,7 @@ message ScoreAdsResponse {
     // BuyerReportingId of the winning Ad.
     // This will be verified with the buyerReportingId in the Ad properties on
     // the browser.
-    string buyer_reporting_id = 18;
+    optional string buyer_reporting_id = 18;
     // BuyerAnd SellerReportingId of the winning Ad.
     // This will be verified with the buyerAndSellerReportingId in the Ad
     // properties on the browser.

diff --git a/api/udf/BUILD b/api/udf/BUILD
@@ -14,6 +14,7 @@
 
 load(
     "@google_privacysandbox_servers_common//src/roma/tools/api_plugin:roma_api.bzl",
+    "declare_doc",
     "declare_roma_api",
     "roma_byob_sdk",
 )
@@ -23,15 +24,14 @@ load("@rules_proto//proto:defs.bzl", "proto_library")
 proto_library(
     name = "generate_bid_proto",
     srcs = ["generate_bid.proto"],
-    visibility = ["//visibility:public"],
+    visibility = ["//api:__subpackages__"],
     deps = [
         "@google_privacysandbox_servers_common//apis/privacysandbox/apis/roma/app_api/v1:options_proto",
     ],
 )
 
 cc_proto_library(
     name = "generate_bid_cc_proto",
-    visibility = ["//visibility:public"],
     deps = [
         ":generate_bid_proto",
     ],
@@ -43,9 +43,46 @@ generate_bid_api = declare_roma_api(
     protos = [":generate_bid_proto"],
 )
 
+generate_bid_guide_intro_text = """
+## Introduction
+
+[Bidding service](https://github.com/privacysandbox/protected-auction-services-docs/blob/2e8d1e9f5f4302ea495c5a1a1a852fd9d01cf607/bidding_auction_services_system_design.md#bidding-service)
+in the
+[B&A TEE based servers](https://github.com/privacysandbox/protected-auction-services-docs/blob/2e8d1e9f5f4302ea495c5a1a1a852fd9d01cf607/bidding_auction_services_system_design.md)
+provides a platform for DSPs to execute their bidding function for each Protected Auction interest
+group in a trusted execution environment. The DSP bidding function could be implemented in
+JavaScript (or WASM driven by JavaScript) conforming to the
+[generateBid spec](https://github.com/privacysandbox/protected-auction-services-docs/blob/2e8d1e9f5f4302ea495c5a1a1a852fd9d01cf607/bidding_auction_services_api.md#generatebid)
+([as also published](https://github.com/WICG/turtledove/blob/a3640067a102fe0759932cd155933f49f4f3c56d/FLEDGE.md#32-on-device-bidding) in
+the Protected Audience web platform explainer).
+
+Bidding service has added support for the execution of a standalone generateBid binary for each
+Protected Auction interest group using the
+[Roma Bring-Your-Own-Binary (BYOB) interface](https://github.com/privacysandbox/protected-auction-services-docs/blob/679f220dd69879de7cc4ba3b7d53cf0c039f5518/roma_bring_your_own_binary.md).
+This enhancement allows developers to implement custom bidding logic beyond the existing JavaScript
+(JS) specifications, offering greater control and efficiency.
+
+This SDK provides the specification, documentation, and tooling for developers of the generateBid binary user-defined function (UDF). Refer to the [Onboarding Guide](generateBid%20Onboarding%20Guide.md) to get started.
+"""
+
 roma_byob_sdk(
-    name = "generate_bid_sdk",
-    srcs = [],
+    name = "generate_bid_byob_sdk",
+    exclude_tools = True,  # TODO(b/373433607): Remove once approval granted.
+    extra_docs = [
+        declare_doc(
+            doc = "//tools/udf/generate_bid/byob_sdk:onboarding_guide",
+            target_subdir = "",
+        ),
+        declare_doc(
+            doc = "//tools/udf/generate_bid/byob_sdk:sample_request_txtpb",
+            target_subdir = "requests",
+        ),
+        declare_doc(
+            doc = "//tools/udf/generate_bid/byob_sdk:sample_request_json",
+            target_subdir = "requests",
+        ),
+    ],
+    guide_intro_text = generate_bid_guide_intro_text,
     roma_app_api = generate_bid_api,
     visibility = ["//visibility:public"],
 )
diff --git a/api/udf/generate_bid.proto b/api/udf/generate_bid.proto
@@ -294,6 +294,8 @@ message PrivateAggregateContribution {
       ' BFE. This is not expected to be sent to the client for single seller'
       ' and server orchestrated multi seller auctions.'
 }];
+
+  int32 ig_idx = 4 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Tells the browser which interest group was this contribution made for.'}];
 }
 
 message PrivateAggregationBucket {

diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf
@@ -169,10 +169,20 @@ module "buyer" {
     MAX_ALLOWED_SIZE_DEBUG_URL_BYTES   = "" # Example: "65536"
     MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000"
 
-    INFERENCE_SIDECAR_BINARY_PATH   = "" # Example: "/server/bin/inference_sidecar_<module_name>"
-    INFERENCE_MODEL_BUCKET_NAME     = "" # Example: "<bucket_name>"
-    INFERENCE_MODEL_CONFIG_PATH     = "" # Example: "model_config.json"
-    INFERENCE_MODEL_FETCH_PERIOD_MS = "" # Example: "60000"
+    INFERENCE_SIDECAR_BINARY_PATH    = "" # Example: "/server/bin/inference_sidecar_<module_name>"
+    INFERENCE_MODEL_BUCKET_NAME      = "" # Example: "<bucket_name>"
+    INFERENCE_MODEL_CONFIG_PATH      = "" # Example: "model_config.json"
+    INFERENCE_MODEL_FETCH_PERIOD_MS  = "" # Example: "60000"
+    INFERENCE_SIDECAR_RUNTIME_CONFIG = "" # Example:
+    # "{
+    #    "num_interop_threads": 4,
+    #    "num_intraop_threads": 4,
+    #    "module_name": "tensorflow_v2_14_0",
+    #    "cpuset": [0, 1, 2, 3],
+    #    "tcmalloc_release_bytes_per_sec": 0,
+    #    "tcmalloc_max_total_thread_cache_bytes": 0,
+    #    "tcmalloc_max_per_cpu_cache_bytes": 0,
+    # }"
 
     # TCMalloc related config parameters.
     # See: https://github.com/google/tcmalloc/blob/master/docs/tuning.md

diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf
@@ -156,8 +156,10 @@ module "seller" {
     SFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND     = "4096"
     SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES                 = "10737418240"
 
-    ENABLE_CHAFFING        = "false"
-    ENABLE_PRIORITY_VECTOR = "false"
+    ENABLE_CHAFFING                     = "false"
+    ENABLE_PRIORITY_VECTOR              = "false"
+    REQUIRE_SCORING_SIGNALS_FOR_SCORING = "true"
+    ALLOW_COMPRESSED_AUCTION_CONFIG     = "false"
   }
   consented_request_s3_bucket = "" # Example: ${name of a s3 bucket}
 }