Add perf metrics for 2.51.0 (#58175)

```
REGRESSION 51.89%: single_client_get_calls_Plasma_Store (THROUGHPUT) regresses from 8378.589542828342 to 4030.5453313124744 in microbenchmark.json
REGRESSION 33.66%: n_n_actor_calls_with_arg_async (THROUGHPUT) regresses from 4105.951978131054 to 2723.9690685388855 in microbenchmark.json
REGRESSION 29.43%: client__tasks_and_put_batch (THROUGHPUT) regresses from 12873.97871447783 to 9085.541921590711 in microbenchmark.json
REGRESSION 27.77%: multi_client_put_calls_Plasma_Store (THROUGHPUT) regresses from 13779.913284159866 to 9952.762154617178 in microbenchmark.json
REGRESSION 26.34%: client__get_calls (THROUGHPUT) regresses from 1129.1409512898194 to 831.689705073893 in microbenchmark.json
REGRESSION 25.07%: multi_client_put_gigabytes (THROUGHPUT) regresses from 36.697734067834084 to 27.49866375110667 in microbenchmark.json
REGRESSION 23.94%: actors_per_second (THROUGHPUT) regresses from 508.9808896382363 to 387.1219957094043 in benchmarks/many_actors.json
REGRESSION 17.22%: 1_1_async_actor_calls_async (THROUGHPUT) regresses from 4826.171895058453 to 3995.0258578261814 in microbenchmark.json
REGRESSION 16.58%: single_client_tasks_async (THROUGHPUT) regresses from 7034.736389002367 to 5868.239300602419 in microbenchmark.json
REGRESSION 14.30%: client__tasks_and_get_batch (THROUGHPUT) regresses from 0.9643252583791863 to 0.8264370292993273 in microbenchmark.json
REGRESSION 14.27%: client__1_1_actor_calls_concurrent (THROUGHPUT) regresses from 1037.1186014627438 to 889.14113884267 in microbenchmark.json
REGRESSION 12.90%: client__1_1_actor_calls_async (THROUGHPUT) regresses from 1014.64288638885 to 883.7347522770161 in microbenchmark.json
REGRESSION 11.37%: client__put_calls (THROUGHPUT) regresses from 805.4069136266919 to 713.82381443796 in microbenchmark.json
REGRESSION 11.02%: 1_1_actor_calls_concurrent (THROUGHPUT) regresses from 5222.99132120111 to 4647.56843532278 in microbenchmark.json
REGRESSION 7.95%: single_client_get_object_containing_10k_refs (THROUGHPUT) regresses from 12.272053704608084 to 11.296427707979271 in microbenchmark.json
REGRESSION 7.86%: client__1_1_actor_calls_sync (THROUGHPUT) regresses from 524.6134993747014 to 483.38098840508496 in microbenchmark.json
REGRESSION 7.26%: single_client_put_calls_Plasma_Store (THROUGHPUT) regresses from 4498.3519827438895 to 4171.572402867286 in microbenchmark.json
REGRESSION 6.47%: single_client_wait_1k_refs (THROUGHPUT) regresses from 4.700920788730696 to 4.396844484606209 in microbenchmark.json
REGRESSION 6.40%: 1_1_async_actor_calls_with_args_async (THROUGHPUT) regresses from 2766.907182403518 to 2589.906655726785 in microbenchmark.json
REGRESSION 5.06%: single_client_put_gigabytes (THROUGHPUT) regresses from 19.30103208209274 to 18.324991353469613 in microbenchmark.json
REGRESSION 1.73%: 1_n_actor_calls_async (THROUGHPUT) regresses from 7474.798821945149 to 7345.613928457275 in microbenchmark.json
REGRESSION 421.24%: dashboard_p99_latency_ms (LATENCY) regresses from 232.641 to 1212.615 in benchmarks/many_pgs.json
REGRESSION 377.77%: dashboard_p95_latency_ms (LATENCY) regresses from 11.336 to 54.16 in benchmarks/many_pgs.json
REGRESSION 306.02%: dashboard_p99_latency_ms (LATENCY) regresses from 749.022 to 3041.184 in benchmarks/many_tasks.json
REGRESSION 162.57%: dashboard_p50_latency_ms (LATENCY) regresses from 11.744 to 30.836 in benchmarks/many_actors.json
REGRESSION 94.47%: dashboard_p95_latency_ms (LATENCY) regresses from 487.355 to 947.76 in benchmarks/many_tasks.json
REGRESSION 35.48%: dashboard_p99_latency_ms (LATENCY) regresses from 49.716 to 67.355 in benchmarks/many_nodes.json
REGRESSION 33.15%: dashboard_p95_latency_ms (LATENCY) regresses from 2876.107 to 3829.61 in benchmarks/many_actors.json
REGRESSION 27.55%: dashboard_p95_latency_ms (LATENCY) regresses from 13.982 to 17.834 in benchmarks/many_nodes.json
REGRESSION 7.56%: time_to_broadcast_1073741824_bytes_to_50_nodes (LATENCY) regresses from 13.777409734000003 to 14.81957527099999 in scalability/object_store.json
REGRESSION 4.48%: 3000_returns_time (LATENCY) regresses from 6.1422604579999955 to 6.417386639000014 in scalability/single_node.json
REGRESSION 3.41%: avg_pg_remove_time_ms (LATENCY) regresses from 1.419495533032749 to 1.4678401576576676 in stress_tests/stress_test_placement_group.json
REGRESSION 1.48%: 10000_get_time (LATENCY) regresses from 25.136106761999997 to 25.508083513999992 in scalability/single_node.json
REGRESSION 0.76%: stage_2_avg_iteration_time (LATENCY) regresses from 36.08304100036621 to 36.358218574523924 in stress_tests/stress_test_many_tasks.json
```

Signed-off-by: kevin <[email protected]>

Author: khluu

release/perf_metrics/benchmarks/many_actors.json

@@ -1,32 +1,31 @@
 {
-    "_dashboard_memory_usage_mb": 108.838912,
+    "_dashboard_memory_usage_mb": 104.996864,
     "_dashboard_test_success": true,
-    "_peak_memory": 4.57,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3786\t2.08GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n6618\t0.81GiB\tpython distributed/test_many_actors.py\n3245\t0.36GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n4000\t0.25GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n1389\t0.11GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4496\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3216\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4498\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru\n3919\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n6383\t0.07GiB\tray::JobSupervisor",
-    "actors_per_second": 508.9808896382363,
+    "_peak_memory": 4.81,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1132\t7.8GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3490\t2.13GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5530\t0.95GiB\tpython distributed/test_many_actors.py\n2919\t0.38GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3716\t0.26GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n585\t0.2GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4196\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3066\t0.1GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3620\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4198\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru",
+    "actors_per_second": 387.1219957094043,
     "num_actors": 10000,
     "perf_metrics": [
         {
             "perf_metric_name": "actors_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 508.9808896382363
+            "perf_metric_value": 387.1219957094043
         },
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 11.744
+            "perf_metric_value": 30.836
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 2876.107
+            "perf_metric_value": 3829.61
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 4160.517
+            "perf_metric_value": 4098.851
         }
     ],
-    "success": "1",
-    "time": 19.64710307121277
+    "time": 25.831650257110596
 }

release/perf_metrics/benchmarks/many_nodes.json

@@ -1,14 +1,14 @@
 {
-    "_dashboard_memory_usage_mb": 98.254848,
+    "_dashboard_memory_usage_mb": 90.251264,
     "_dashboard_test_success": true,
-    "_peak_memory": 2.33,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3580\t0.52GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3098\t0.28GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5630\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3796\t0.13GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n1458\t0.11GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4276\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3020\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3713\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4278\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru\n5857\t0.09GiB\tray::StateAPIGeneratorActor.start",
+    "_peak_memory": 2.42,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3297\t0.64GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2878\t0.29GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5295\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3510\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n1172\t0.13GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3996\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n2813\t0.1GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3998\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru\n3427\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n5522\t0.08GiB\tray::StateAPIGeneratorActor.start",
     "num_tasks": 1000,
     "perf_metrics": [
         {
             "perf_metric_name": "tasks_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 190.09519026949954
+            "perf_metric_value": 355.96107999624206
         },
         {
             "perf_metric_name": "used_cpus_by_deadline",
@@ -18,21 +18,20 @@
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 7.136
+            "perf_metric_value": 6.475
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 13.982
+            "perf_metric_value": 17.834
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 49.716
+            "perf_metric_value": 67.355
         }
     ],
-    "success": "1",
-    "tasks_per_second": 190.09519026949954,
-    "time": 305.26052236557007,
+    "tasks_per_second": 355.96107999624206,
+    "time": 302.80929589271545,
     "used_cpus": 250.0
 }

release/perf_metrics/benchmarks/many_pgs.json

@@ -1,32 +1,31 @@
 {
-    "_dashboard_memory_usage_mb": 90.76736,
+    "_dashboard_memory_usage_mb": 99.979264,
     "_dashboard_test_success": true,
-    "_peak_memory": 2.81,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1393\t7.16GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3778\t0.97GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3247\t0.42GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5319\t0.36GiB\tpython distributed/test_many_pgs.py\n600\t0.2GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4001\t0.11GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3044\t0.11GiB\t/app/go/infra/activityprobe/activityprobe ray --port=5903 --metrics_server_port=9092 --raylet_addr=l\n4486\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3232\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4488\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru",
+    "_peak_memory": 2.94,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1126\t7.66GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3490\t1.1GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5004\t0.36GiB\tpython distributed/test_many_pgs.py\n3037\t0.34GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n583\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4206\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3720\t0.11GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n2924\t0.1GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4208\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru\n3623\t0.08GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash",
     "num_pgs": 1000,
     "perf_metrics": [
         {
             "perf_metric_name": "pgs_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 12.47149444972938
+            "perf_metric_value": 17.897951502183457
         },
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 4.935
+            "perf_metric_value": 4.128
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 11.336
+            "perf_metric_value": 54.16
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 232.641
+            "perf_metric_value": 1212.615
         }
     ],
-    "pgs_per_second": 12.47149444972938,
-    "success": "1",
-    "time": 80.18285250663757
+    "pgs_per_second": 17.897951502183457,
+    "time": 55.872315883636475
 }

release/perf_metrics/benchmarks/many_tasks.json

@@ -1,14 +1,14 @@
 {
-    "_dashboard_memory_usage_mb": 95.678464,
+    "_dashboard_memory_usage_mb": 104.124416,
     "_dashboard_test_success": true,
-    "_peak_memory": 4.01,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3780\t1.1GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5801\t0.75GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3998\t0.48GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3294\t0.27GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n4001\t0.18GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n4493\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n1403\t0.1GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3184\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3916\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4495\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru",
+    "_peak_memory": 6.24,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3751\t2.0GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3524\t1.95GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5011\t0.76GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3015\t0.27GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n582\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n1163\t0.11GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3754\t0.11GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n4237\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3147\t0.1GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3654\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash",
     "num_tasks": 10000,
     "perf_metrics": [
         {
             "perf_metric_name": "tasks_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 368.5098005212305
+            "perf_metric_value": 571.2270630108624
         },
         {
             "perf_metric_name": "used_cpus_by_deadline",
@@ -18,21 +18,20 @@
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 6.4
+            "perf_metric_value": 6.284
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 487.355
+            "perf_metric_value": 947.76
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 749.022
+            "perf_metric_value": 3041.184
         }
     ],
-    "success": "1",
-    "tasks_per_second": 368.5098005212305,
-    "time": 327.1363203525543,
+    "tasks_per_second": 571.2270630108624,
+    "time": 317.5061733722687,
     "used_cpus": 2500.0
 }

release/perf_metrics/metadata.json

@@ -1 +1 @@
-{"release_version": "2.50.0"}
+{"release_version": "2.51.0"}