diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index c8968cb056..938240ab1e 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -144,6 +144,10 @@ jobs: working-directory: ./tests run: make create-and-setup-vm + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true + - name: Run Tests working-directory: ./tests run: | @@ -164,6 +168,7 @@ jobs: if: always() run: | bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true make delete-gke-cluster rm -rf scripts/vars.env diff --git a/tests/results/dp-perf/edge/edge-oss.md b/tests/results/dp-perf/edge/edge-oss.md index ee23694a87..24411b0755 100644 --- a/tests/results/dp-perf/edge/edge-oss.md +++ b/tests/results/dp-perf/edge/edge-oss.md @@ -6,16 +6,16 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851332Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 @@ -23,9 +23,9 @@ GKE Cluster: ## Test1: Running latte path based routing ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 569.726µs -Latencies [min, mean, 50, 90, 95, 99, max] 492.479µs, 670.385µs, 659.036µs, 746.275µs, 777.873µs, 857.407µs, 10.667ms +Requests [total, rate, throughput] 30000, 1000.03, 1000.01 +Duration [total, attack, wait] 30s, 29.999s, 860.394µs +Latencies [min, mean, 50, 90, 95, 99, max] 700.394µs, 937.732µs, 918.398µs, 1.047ms, 1.092ms, 1.23ms, 14.431ms Bytes In [total, mean] 4800000, 160.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% @@ -36,22 +36,22 @@ Error Set: ## Test2: Running coffee header based routing ```text -Requests [total, rate, throughput] 29999, 1000.00, 999.98 -Duration [total, attack, wait] 30s, 29.999s, 665.107µs -Latencies [min, mean, 50, 90, 95, 99, max] 518.165µs, 707.025µs, 693.839µs, 792.941µs, 827.269µs, 914.615µs, 9.399ms -Bytes In [total, mean] 4829839, 161.00 +Requests [total, rate, throughput] 30000, 1000.06, 1000.03 +Duration [total, attack, wait] 29.999s, 29.998s, 912.253µs +Latencies [min, mean, 50, 90, 95, 99, max] 729.21µs, 969.285µs, 954.2µs, 1.069ms, 1.116ms, 1.246ms, 17.953ms +Bytes In [total, mean] 4830000, 161.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:29999 +Status Codes [code:count] 200:30000 Error Set: ``` ## Test3: Running coffee query based routing ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.01 -Duration [total, attack, wait] 30s, 29.999s, 715.919µs -Latencies [min, mean, 50, 90, 95, 99, max] 535.068µs, 708.655µs, 696.175µs, 794.741µs, 829.728µs, 926.641µs, 9.422ms +Requests [total, rate, throughput] 30000, 1000.02, 999.99 +Duration [total, attack, wait] 30s, 29.999s, 916.998µs +Latencies [min, mean, 50, 90, 95, 99, max] 699.499µs, 968.324µs, 949.334µs, 1.071ms, 1.112ms, 1.23ms, 15.731ms Bytes In [total, mean] 5070000, 169.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% @@ -62,9 +62,9 @@ Error Set: ## Test4: Running tea GET method based routing ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 689.244µs -Latencies [min, mean, 50, 90, 95, 99, max] 517.044µs, 689.83µs, 678.3µs, 768.738µs, 802.493µs, 884.763µs, 13.123ms +Requests [total, rate, throughput] 30000, 1000.04, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 983.302µs +Latencies [min, mean, 50, 90, 95, 99, max] 701.738µs, 959.004µs, 940.79µs, 1.063ms, 1.105ms, 1.252ms, 21.194ms Bytes In [total, mean] 4740000, 158.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% @@ -75,9 +75,9 @@ Error Set: ## Test5: Running tea POST method based routing ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.01 -Duration [total, attack, wait] 30s, 29.999s, 618.418µs -Latencies [min, mean, 50, 90, 95, 99, max] 506.217µs, 700.343µs, 688.984µs, 785.078µs, 815.876µs, 898.036µs, 9.243ms +Requests [total, rate, throughput] 30000, 1000.04, 1000.01 +Duration [total, attack, wait] 30s, 29.999s, 931.139µs +Latencies [min, mean, 50, 90, 95, 99, max] 721.208µs, 948.397µs, 934.072µs, 1.053ms, 1.1ms, 1.244ms, 10.389ms Bytes In [total, mean] 4740000, 158.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% diff --git a/tests/results/dp-perf/edge/edge-plus.md b/tests/results/dp-perf/edge/edge-plus.md index 33e8224482..f20af13229 100644 --- a/tests/results/dp-perf/edge/edge-plus.md +++ b/tests/results/dp-perf/edge/edge-plus.md @@ -6,14 +6,14 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 - RAM per node: 65851340Ki - Max pods per node: 110 @@ -23,23 +23,23 @@ GKE Cluster: ## Test1: Running latte path based routing ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 662.781µs -Latencies [min, mean, 50, 90, 95, 99, max] 484.55µs, 685.236µs, 667.104µs, 770.782µs, 811.42µs, 914.291µs, 14.722ms -Bytes In [total, mean] 4830000, 161.00 +Requests [total, rate, throughput] 29999, 1000.00, 999.97 +Duration [total, attack, wait] 30s, 29.999s, 791.665µs +Latencies [min, mean, 50, 90, 95, 99, max] 675.837µs, 931.514µs, 900.171µs, 1.028ms, 1.083ms, 1.294ms, 23.359ms +Bytes In [total, mean] 4739842, 158.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:30000 +Status Codes [code:count] 200:29999 Error Set: ``` ## Test2: Running coffee header based routing ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.01 -Duration [total, attack, wait] 30s, 29.999s, 762.608µs -Latencies [min, mean, 50, 90, 95, 99, max] 518.259µs, 717.123µs, 697.898µs, 803.297µs, 846.307µs, 967.712µs, 18.472ms -Bytes In [total, mean] 4860000, 162.00 +Requests [total, rate, throughput] 30000, 1000.03, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 829.798µs +Latencies [min, mean, 50, 90, 95, 99, max] 699.872µs, 954.024µs, 924.522µs, 1.058ms, 1.106ms, 1.279ms, 207.596ms +Bytes In [total, mean] 4770000, 159.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 @@ -49,10 +49,10 @@ Error Set: ## Test3: Running coffee query based routing ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 686.373µs -Latencies [min, mean, 50, 90, 95, 99, max] 492.406µs, 724.55µs, 708.483µs, 817.376µs, 858.736µs, 976.771µs, 11.812ms -Bytes In [total, mean] 5100000, 170.00 +Requests [total, rate, throughput] 30000, 1000.06, 1000.02 +Duration [total, attack, wait] 29.999s, 29.998s, 1.166ms +Latencies [min, mean, 50, 90, 95, 99, max] 668.81µs, 960.172µs, 941.911µs, 1.07ms, 1.118ms, 1.291ms, 11.068ms +Bytes In [total, mean] 5010000, 167.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 @@ -62,23 +62,23 @@ Error Set: ## Test4: Running tea GET method based routing ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 695.346µs -Latencies [min, mean, 50, 90, 95, 99, max] 529.34µs, 718.101µs, 700.968µs, 809.033µs, 850.203µs, 954.285µs, 12.013ms -Bytes In [total, mean] 4770000, 159.00 +Requests [total, rate, throughput] 29999, 1000.00, 999.97 +Duration [total, attack, wait] 30s, 29.999s, 1.003ms +Latencies [min, mean, 50, 90, 95, 99, max] 705.934µs, 980.306µs, 962.806µs, 1.1ms, 1.159ms, 1.374ms, 18.465ms +Bytes In [total, mean] 4679844, 156.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:30000 +Status Codes [code:count] 200:29999 Error Set: ``` ## Test5: Running tea POST method based routing ```text -Requests [total, rate, throughput] 30000, 1000.01, 999.99 -Duration [total, attack, wait] 30s, 30s, 683.707µs -Latencies [min, mean, 50, 90, 95, 99, max] 510.765µs, 709.274µs, 692.005µs, 795.696µs, 836.686µs, 946.19µs, 15.089ms -Bytes In [total, mean] 4770000, 159.00 +Requests [total, rate, throughput] 30000, 1000.02, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 843.32µs +Latencies [min, mean, 50, 90, 95, 99, max] 727.931µs, 984.586µs, 965.124µs, 1.107ms, 1.164ms, 1.329ms, 13.12ms +Bytes In [total, mean] 4680000, 156.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 diff --git a/tests/results/ngf-upgrade/edge/edge-oss.md b/tests/results/ngf-upgrade/edge/edge-oss.md index b91e00b68a..c7d9b1712f 100644 --- a/tests/results/ngf-upgrade/edge/edge-oss.md +++ b/tests/results/ngf-upgrade/edge/edge-oss.md @@ -6,46 +6,16 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851332Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 - -## Test: Send http /coffee traffic - -```text -Requests [total, rate, throughput] 6000, 100.02, 100.01 -Duration [total, attack, wait] 59.992s, 59.991s, 864.529µs -Latencies [min, mean, 50, 90, 95, 99, max] 470.608µs, 866.796µs, 864.021µs, 991.562µs, 1.037ms, 1.156ms, 10.317ms -Bytes In [total, mean] 967993, 161.33 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:6000 -Error Set: -``` - -![http-oss.png](http-oss.png) - -## Test: Send https /tea traffic - -```text -Requests [total, rate, throughput] 6000, 100.02, 100.01 -Duration [total, attack, wait] 59.992s, 59.991s, 850.998µs -Latencies [min, mean, 50, 90, 95, 99, max] 455.17µs, 901.793µs, 886.37µs, 1.017ms, 1.06ms, 1.177ms, 10.281ms -Bytes In [total, mean] 931993, 155.33 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:6000 -Error Set: -``` - -![https-oss.png](https-oss.png) diff --git a/tests/results/ngf-upgrade/edge/edge-plus.md b/tests/results/ngf-upgrade/edge/edge-plus.md index 2bf3b4d5f8..0f1850bf93 100644 --- a/tests/results/ngf-upgrade/edge/edge-plus.md +++ b/tests/results/ngf-upgrade/edge/edge-plus.md @@ -6,46 +6,16 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 - RAM per node: 65851340Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 - -## Test: Send http /coffee traffic - -```text -Requests [total, rate, throughput] 6000, 100.02, 100.02 -Duration [total, attack, wait] 59.99s, 59.989s, 855.743µs -Latencies [min, mean, 50, 90, 95, 99, max] 635.422µs, 851.551µs, 832.349µs, 964.217µs, 1.017ms, 1.176ms, 9.726ms -Bytes In [total, mean] 972000, 162.00 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:6000 -Error Set: -``` - -![http-plus.png](http-plus.png) - -## Test: Send https /tea traffic - -```text -Requests [total, rate, throughput] 6000, 100.02, 100.02 -Duration [total, attack, wait] 59.991s, 59.99s, 673.229µs -Latencies [min, mean, 50, 90, 95, 99, max] 439.111µs, 903.895µs, 887.003µs, 1.057ms, 1.116ms, 1.284ms, 9.699ms -Bytes In [total, mean] 936000, 156.00 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:6000 -Error Set: -``` - -![https-plus.png](https-plus.png) diff --git a/tests/results/reconfig/edge/edge-oss.md b/tests/results/reconfig/edge/edge-oss.md index a9a5e8f46b..064218ce41 100644 --- a/tests/results/reconfig/edge/edge-oss.md +++ b/tests/results/reconfig/edge/edge-oss.md @@ -6,201 +6,135 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851332Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 ## Test 1: Resources exist before startup - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 3s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 101ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 9s ### Event Batch Processing -- Event Batch Total: 5 -- Event Batch Processing Average Time: 53ms +- Event Batch Total: 10 +- Event Batch Processing Average Time: 4ms - Event Batch Processing distribution: - - 500.0ms: 5 - - 1000.0ms: 5 - - 5000.0ms: 5 - - 10000.0ms: 5 - - 30000.0ms: 5 - - +Infms: 5 + - 500.0ms: 10 + - 1000.0ms: 10 + - 5000.0ms: 10 + - 10000.0ms: 10 + - 30000.0ms: 10 + - +Infms: 10 ### NGINX Error Logs ## Test 1: Resources exist before startup - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 3s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 88ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 6s ### Event Batch Processing -- Event Batch Total: 6 -- Event Batch Processing Average Time: 45ms +- Event Batch Total: 10 +- Event Batch Processing Average Time: 6ms - Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 + - 500.0ms: 10 + - 1000.0ms: 10 + - 5000.0ms: 10 + - 10000.0ms: 10 + - 30000.0ms: 10 + - +Infms: 10 ### NGINX Error Logs -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 8s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 63 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 63 - - 1000.0ms: 63 - - 5000.0ms: 63 - - 10000.0ms: 63 - - 30000.0ms: 63 - - +Infms: 63 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 23s ### Event Batch Processing -- Event Batch Total: 337 -- Event Batch Processing Average Time: 23ms +- Event Batch Total: 296 +- Event Batch Processing Average Time: 19ms - Event Batch Processing distribution: - - 500.0ms: 337 - - 1000.0ms: 337 - - 5000.0ms: 337 - - 10000.0ms: 337 - - 30000.0ms: 337 - - +Infms: 337 + - 500.0ms: 296 + - 1000.0ms: 296 + - 5000.0ms: 296 + - 10000.0ms: 296 + - 30000.0ms: 296 + - +Infms: 296 ### NGINX Error Logs +2025/05/20 19:36:40 [emerg] 8#8: pread() returned only 0 bytes instead of 4085 in /etc/nginx/conf.d/http.conf:1160 +2025/05/20 19:36:42 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:2955 +2025/05/20 19:36:43 [emerg] 8#8: unknown directive "roxy_set_header" in /etc/nginx/conf.d/http.conf:1081 -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 44s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 343 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 343 - - 1000.0ms: 343 - - 5000.0ms: 343 - - 10000.0ms: 343 - - 30000.0ms: 343 - - +Infms: 343 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 122s ### Event Batch Processing -- Event Batch Total: 1689 -- Event Batch Processing Average Time: 25ms +- Event Batch Total: 1405 +- Event Batch Processing Average Time: 28ms - Event Batch Processing distribution: - - 500.0ms: 1689 - - 1000.0ms: 1689 - - 5000.0ms: 1689 - - 10000.0ms: 1689 - - 30000.0ms: 1689 - - +Infms: 1689 + - 500.0ms: 1405 + - 1000.0ms: 1405 + - 5000.0ms: 1405 + - 10000.0ms: 1405 + - 30000.0ms: 1405 + - +Infms: 1405 ### NGINX Error Logs +2025/05/20 19:47:56 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:1608 +2025/05/20 19:47:58 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:2628 +2025/05/20 19:47:59 [emerg] 8#8: pread() returned only 0 bytes instead of 4093 in /etc/nginx/conf.d/http.conf:1309 +2025/05/20 19:47:59 [emerg] 8#8: pread() returned only 0 bytes instead of 2351 in /etc/nginx/conf.d/http.conf:2799 +2025/05/20 19:48:02 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:3947 +2025/05/20 19:48:03 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:766 +2025/05/20 19:48:04 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:4492 +2025/05/20 19:48:07 [emerg] 8#8: pread() returned only 0 bytes instead of 4091 in /etc/nginx/conf.d/http.conf:2140 +2025/05/20 19:48:08 [emerg] 8#8: pread() returned only 0 bytes instead of 4086 in /etc/nginx/conf.d/http.conf:2509 +2025/05/20 19:48:10 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:6672 +2025/05/20 19:48:12 [emerg] 8#8: pread() returned only 0 bytes instead of 4092 in /etc/nginx/conf.d/http.conf:1961 +2025/05/20 19:48:12 [emerg] 8#8: pread() returned only 0 bytes instead of 4093 in /etc/nginx/conf.d/http.conf:6461 +2025/05/20 19:48:16 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:8961 +2025/05/20 19:48:17 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:9192 +2025/05/20 19:48:19 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:3363 +2025/05/20 19:48:20 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:4683 +2025/05/20 19:48:22 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:11044 +2025/05/20 19:48:23 [emerg] 8#8: pread() returned only 0 bytes instead of 4089 in /etc/nginx/conf.d/http.conf:1517 +2025/05/20 19:48:23 [emerg] 8#8: pread() returned only 0 bytes instead of 4083 in /etc/nginx/conf.d/http.conf:4254 +2025/05/20 19:48:25 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:412 +2025/05/20 19:48:25 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:12547 +2025/05/20 19:48:26 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:12726 +2025/05/20 19:48:27 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:13162 +2025/05/20 19:48:29 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:13746 +2025/05/20 19:48:29 [emerg] 8#8: pread() returned only 0 bytes instead of 4089 in /etc/nginx/conf.d/http.conf:5285 +2025/05/20 19:48:32 [emerg] 8#8: pread() returned only 0 bytes instead of 4088 in /etc/nginx/conf.d/http.conf:3293 +2025/05/20 19:48:34 [emerg] 8#8: pread() returned only 0 bytes instead of 4085 in /etc/nginx/conf.d/http.conf:1285 +2025/05/20 19:48:35 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:16046 - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 64 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 64 - - 1000.0ms: 64 - - 5000.0ms: 64 - - 10000.0ms: 64 - - 30000.0ms: 64 - - +Infms: 64 - -### Event Batch Processing - -- Event Batch Total: 321 -- Event Batch Processing Average Time: 25ms -- Event Batch Processing distribution: - - 500.0ms: 321 - - 1000.0ms: 321 - - 5000.0ms: 321 - - 10000.0ms: 321 - - 30000.0ms: 321 - - +Infms: 321 - -### NGINX Error Logs - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 342 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 342 - - 1000.0ms: 342 - - 5000.0ms: 342 - - 10000.0ms: 342 - - 30000.0ms: 342 - - +Infms: 342 - -### Event Batch Processing - -- Event Batch Total: 1639 -- Event Batch Processing Average Time: 26ms -- Event Batch Processing distribution: - - 500.0ms: 1639 - - 1000.0ms: 1639 - - 5000.0ms: 1639 - - 10000.0ms: 1639 - - 30000.0ms: 1639 - - +Infms: 1639 - -### NGINX Error Logs diff --git a/tests/results/reconfig/edge/edge-plus.md b/tests/results/reconfig/edge/edge-plus.md index b339fb3c0d..9e61b73645 100644 --- a/tests/results/reconfig/edge/edge-plus.md +++ b/tests/results/reconfig/edge/edge-plus.md @@ -6,14 +6,14 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 - RAM per node: 65851340Ki - Max pods per node: 110 @@ -22,186 +22,88 @@ GKE Cluster: ## Test 1: Resources exist before startup - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 4s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 100ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 28s ### Event Batch Processing -- Event Batch Total: 6 -- Event Batch Processing Average Time: 52ms +- Event Batch Total: 8 +- Event Batch Processing Average Time: 28ms - Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 + - 500.0ms: 8 + - 1000.0ms: 8 + - 5000.0ms: 8 + - 10000.0ms: 8 + - 30000.0ms: 8 + - +Infms: 8 ### NGINX Error Logs ## Test 1: Resources exist before startup - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 4s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 100ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 6s ### Event Batch Processing -- Event Batch Total: 6 -- Event Batch Processing Average Time: 53ms +- Event Batch Total: 9 +- Event Batch Processing Average Time: 26ms - Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 + - 500.0ms: 9 + - 1000.0ms: 9 + - 5000.0ms: 9 + - 10000.0ms: 9 + - 30000.0ms: 9 + - +Infms: 9 ### NGINX Error Logs -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 8s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 47 -- NGINX Reload Average Time: 148ms -- Reload distribution: - - 500.0ms: 47 - - 1000.0ms: 47 - - 5000.0ms: 47 - - 10000.0ms: 47 - - 30000.0ms: 47 - - +Infms: 47 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 27s ### Event Batch Processing -- Event Batch Total: 322 -- Event Batch Processing Average Time: 25ms +- Event Batch Total: 227 +- Event Batch Processing Average Time: 37ms - Event Batch Processing distribution: - - 500.0ms: 322 - - 1000.0ms: 322 - - 5000.0ms: 322 - - 10000.0ms: 322 - - 30000.0ms: 322 - - +Infms: 322 + - 500.0ms: 216 + - 1000.0ms: 227 + - 5000.0ms: 227 + - 10000.0ms: 227 + - 30000.0ms: 227 + - +Infms: 227 ### NGINX Error Logs -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 20s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 117 -- NGINX Reload Average Time: 150ms -- Reload distribution: - - 500.0ms: 117 - - 1000.0ms: 117 - - 5000.0ms: 117 - - 10000.0ms: 117 - - 30000.0ms: 117 - - +Infms: 117 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 144s ### Event Batch Processing -- Event Batch Total: 1460 -- Event Batch Processing Average Time: 14ms +- Event Batch Total: 1098 +- Event Batch Processing Average Time: 44ms - Event Batch Processing distribution: - - 500.0ms: 1460 - - 1000.0ms: 1460 - - 5000.0ms: 1460 - - 10000.0ms: 1460 - - 30000.0ms: 1460 - - +Infms: 1460 + - 500.0ms: 1059 + - 1000.0ms: 1080 + - 5000.0ms: 1098 + - 10000.0ms: 1098 + - 30000.0ms: 1098 + - +Infms: 1098 ### NGINX Error Logs -2025/03/15 17:00:26 [emerg] 48#48: invalid instance state file "/var/lib/nginx/state/nginx-mgmt-state" - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 46 -- NGINX Reload Average Time: 133ms -- Reload distribution: - - 500.0ms: 46 - - 1000.0ms: 46 - - 5000.0ms: 46 - - 10000.0ms: 46 - - 30000.0ms: 46 - - +Infms: 46 - -### Event Batch Processing - -- Event Batch Total: 291 -- Event Batch Processing Average Time: 28ms -- Event Batch Processing distribution: - - 500.0ms: 291 - - 1000.0ms: 291 - - 5000.0ms: 291 - - 10000.0ms: 291 - - 30000.0ms: 291 - - +Infms: 291 - -### NGINX Error Logs - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 258 -- NGINX Reload Average Time: 132ms -- Reload distribution: - - 500.0ms: 258 - - 1000.0ms: 258 - - 5000.0ms: 258 - - 10000.0ms: 258 - - 30000.0ms: 258 - - +Infms: 258 - -### Event Batch Processing - -- Event Batch Total: 1501 -- Event Batch Processing Average Time: 29ms -- Event Batch Processing distribution: - - 500.0ms: 1501 - - 1000.0ms: 1501 - - 5000.0ms: 1501 - - 10000.0ms: 1501 - - 30000.0ms: 1501 - - +Infms: 1501 - -### NGINX Error Logs diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-oss.png b/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-oss.png index 08cc64bf7d..6f02e7f6ef 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-oss.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-plus.png b/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-plus.png index d8a73bd8e6..9e10a565b4 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-plus.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/cpu-plus.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/memory-oss.png b/tests/results/scale/edge/TestScale_HTTPRoutes/memory-oss.png index e940514c20..c18f5acd9a 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/memory-oss.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/memory-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/memory-plus.png b/tests/results/scale/edge/TestScale_HTTPRoutes/memory-plus.png index 1d2c2028a9..64565f6e6a 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/memory-plus.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/memory-plus.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-oss.png b/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-oss.png index cbb86f3c40..bb61879007 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-oss.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-plus.png b/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-plus.png index fa76f68e10..6e604af203 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-plus.png and b/tests/results/scale/edge/TestScale_HTTPRoutes/ttr-plus.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-oss.png b/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-oss.png index d299f397c9..9511f3e378 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-oss.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-plus.png b/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-plus.png index e75dd0cd66..f93707444c 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-plus.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/cpu-plus.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/memory-oss.png b/tests/results/scale/edge/TestScale_HTTPSListeners/memory-oss.png index 24f13fa2a4..616896880e 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/memory-oss.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/memory-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/memory-plus.png b/tests/results/scale/edge/TestScale_HTTPSListeners/memory-plus.png index 5b5edd2b05..b9e9144dcf 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/memory-plus.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/memory-plus.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-oss.log b/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-oss.log index 793908c50a..bf7854bec7 100644 --- a/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-oss.log +++ b/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-oss.log @@ -1 +1,18 @@ -{"level":"debug","ts":"2024-10-15T19:05:58Z","logger":"controller-runtime.healthz","msg":"healthz check failed","checker":"readyz","error":"nginx has not yet become ready to accept traffic"} +{"level":"debug","ts":"2025-05-20T18:22:12Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:22:57Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:22:59Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:03Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:08Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:18Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:19Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:20Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:23Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:28Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:29Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:32Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:34Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:43Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:44Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:23:46Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-plus.log b/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-plus.log index b75fafd9d2..6b52f736bc 100644 --- a/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-plus.log +++ b/tests/results/scale/edge/TestScale_HTTPSListeners/ngf-plus.log @@ -1,3 +1,14 @@ -{"level":"debug","ts":"2024-10-15T18:17:33Z","logger":"controller-runtime.healthz","msg":"healthz check failed","checker":"readyz","error":"nginx has not yet become ready to accept traffic"} -{"level":"debug","ts":"2024-10-15T18:17:33Z","logger":"controller-runtime.healthz","msg":"healthz check failed","checker":"readyz","error":"nginx has not yet become ready to accept traffic"} -{"level":"debug","ts":"2024-10-15T18:19:29Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:18:31Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:27Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:30Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:38Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:39Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:41Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:42Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:47Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:19:48Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:20:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:20:03Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:20:05Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:20:47Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:20:49Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/nginx-plus.log b/tests/results/scale/edge/TestScale_HTTPSListeners/nginx-plus.log index 93f206ec65..611aa1b595 100644 --- a/tests/results/scale/edge/TestScale_HTTPSListeners/nginx-plus.log +++ b/tests/results/scale/edge/TestScale_HTTPSListeners/nginx-plus.log @@ -1,15 +1,171 @@ -2025/03/15 19:02:20 [error] 163#163: *26 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" -2025/03/15 19:02:21 [error] 197#197: *38 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" -2025/03/15 19:02:21 [error] 232#232: *51 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" -2025/03/15 19:02:23 [error] 334#334: *88 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" -2025/03/15 19:02:32 [error] 810#810: *263 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" -2025/03/15 19:02:33 [error] 844#844: *276 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 21.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-21_80/", host: "21.example.com" -2025/03/15 19:02:46 [error] 1252#1252: *436 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 33.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-33_80/", host: "33.example.com" -2025/03/15 19:03:01 [error] 1626#1626: *598 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 44.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-44_80/", host: "44.example.com" -2025/03/15 19:03:03 [error] 1660#1660: *613 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 45.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-45_80/", host: "45.example.com" -2025/03/15 19:03:05 [error] 1694#1694: *628 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 46.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-46_80/", host: "46.example.com" -2025/03/15 19:03:07 [error] 1728#1728: *643 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" -2025/03/15 19:03:10 [error] 1796#1796: *675 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" -2025/03/15 19:03:22 [error] 2000#2000: *772 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" -2025/03/15 19:03:24 [error] 2034#2034: *788 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" -2025/03/15 19:03:35 [error] 2204#2204: *871 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:19:18 [error] 147#147: *68 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/05/20 20:19:18 [error] 148#148: *69 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/05/20 20:19:18 [error] 150#150: *70 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/05/20 20:19:18 [error] 151#151: *71 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/05/20 20:19:19 [error] 183#183: *85 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:19:19 [error] 184#184: *86 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:19:19 [error] 186#186: *87 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:19:19 [error] 187#187: *88 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:19:19 [error] 219#219: *103 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:19:20 [error] 220#220: *104 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:19:20 [error] 222#222: *105 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:19:20 [error] 223#223: *106 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:19:20 [error] 255#255: *122 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:19:20 [error] 256#256: *123 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:19:21 [error] 258#258: *124 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:19:21 [error] 259#259: *125 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:19:21 [error] 291#291: *143 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:19:21 [error] 292#292: *144 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:19:21 [error] 294#294: *145 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:19:22 [error] 295#295: *146 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:19:22 [error] 296#296: *147 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:19:22 [error] 329#329: *165 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:19:22 [error] 330#330: *166 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:19:22 [error] 332#332: *167 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:19:23 [error] 333#333: *168 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:19:23 [error] 334#334: *169 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:19:23 [error] 365#365: *204 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:19:23 [error] 366#366: *205 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:19:23 [error] 368#368: *206 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:19:24 [error] 369#369: *207 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:19:24 [error] 370#370: *208 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:19:24 [error] 401#401: *228 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:19:24 [error] 402#402: *229 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:19:24 [error] 404#404: *230 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:19:25 [error] 405#405: *231 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:19:25 [error] 406#406: *232 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:19:25 [error] 441#441: *253 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:19:25 [error] 437#437: *254 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:19:25 [error] 438#438: *255 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:19:26 [error] 440#440: *256 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:19:26 [error] 442#442: *257 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:19:26 [error] 473#473: *271 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:19:26 [error] 474#474: *272 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:19:27 [error] 476#476: *273 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:19:27 [error] 509#509: *296 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:19:27 [error] 510#510: *297 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:19:27 [error] 512#512: *298 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:19:28 [error] 513#513: *299 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:19:28 [error] 545#545: *323 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:19:28 [error] 546#546: *324 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:19:28 [error] 548#548: *325 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:19:28 [error] 549#549: *326 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:19:29 [error] 581#581: *351 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:19:29 [error] 582#582: *352 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:19:29 [error] 584#584: *353 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:19:29 [error] 585#585: *354 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:19:30 [error] 617#617: *381 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:19:30 [error] 618#618: *382 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:19:30 [error] 620#620: *383 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:19:30 [error] 621#621: *384 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:19:30 [error] 622#622: *385 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:19:31 [error] 653#653: *412 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:19:31 [error] 654#654: *413 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:19:31 [error] 656#656: *415 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:19:31 [error] 657#657: *416 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:19:31 [error] 658#658: *417 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:19:32 [error] 691#691: *446 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:19:32 [error] 692#692: *447 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:19:33 [error] 694#694: *448 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:19:33 [error] 695#695: *449 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:19:33 [error] 696#696: *465 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:19:34 [error] 730#730: *494 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:19:34 [error] 727#727: *495 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:19:34 [error] 728#728: *496 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:19:34 [error] 731#731: *497 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:19:34 [error] 732#732: *498 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:19:35 [error] 763#763: *529 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:19:35 [error] 764#764: *530 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:19:35 [error] 766#766: *531 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:19:35 [error] 767#767: *532 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:19:36 [error] 799#799: *564 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:19:36 [error] 800#800: *565 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:19:36 [error] 802#802: *566 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:19:36 [error] 803#803: *567 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:19:36 [error] 804#804: *569 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:19:37 [error] 837#837: *603 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/05/20 20:19:37 [error] 838#838: *604 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/05/20 20:19:37 [error] 840#840: *605 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/05/20 20:19:37 [error] 841#841: *606 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/05/20 20:19:38 [error] 842#842: *607 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/05/20 20:19:42 [error] 983#983: *740 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 24.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-24_80/", host: "24.example.com" +2025/05/20 20:19:47 [error] 1165#1165: *946 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 29.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-29_80/", host: "29.example.com" +2025/05/20 20:19:49 [error] 1201#1201: *993 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 30.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-30_80/", host: "30.example.com" +2025/05/20 20:19:50 [error] 1237#1237: *1030 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 31.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-31_80/", host: "31.example.com" +2025/05/20 20:19:51 [error] 1273#1273: *1076 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 32.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-32_80/", host: "32.example.com" +2025/05/20 20:19:52 [error] 1311#1311: *1124 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 33.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/05/20 20:19:53 [error] 1347#1347: *1181 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 34.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-34_80/", host: "34.example.com" +2025/05/20 20:19:54 [error] 1383#1383: *1230 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 35.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/05/20 20:19:56 [error] 1419#1419: *1273 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 36.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/05/20 20:19:57 [error] 1455#1455: *1325 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 37.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/05/20 20:19:58 [error] 1491#1491: *1370 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 38.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/05/20 20:19:58 [error] 1492#1492: *1382 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 38.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/05/20 20:20:00 [error] 1527#1527: *1421 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 39.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/05/20 20:20:01 [error] 1563#1563: *1476 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 40.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/05/20 20:20:03 [error] 1601#1601: *1546 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 41.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/05/20 20:20:04 [error] 1638#1638: *1591 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 42.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/05/20 20:20:04 [error] 1639#1639: *1605 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 42.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/05/20 20:20:06 [error] 1674#1674: *1648 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 43.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/05/20 20:20:06 [error] 1675#1675: *1660 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 43.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/05/20 20:20:07 [error] 1712#1712: *1703 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 44.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/05/20 20:20:07 [error] 1713#1713: *1718 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 44.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/05/20 20:20:08 [error] 1748#1748: *1762 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 45.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/05/20 20:20:09 [error] 1749#1749: *1774 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 45.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/05/20 20:20:10 [error] 1784#1784: *1818 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 46.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/05/20 20:20:11 [error] 1785#1785: *1831 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 46.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/05/20 20:20:13 [error] 1823#1823: *1876 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:20:13 [error] 1822#1822: *1888 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:20:13 [error] 1825#1825: *1917 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:20:15 [error] 1858#1858: *1955 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:20:15 [error] 1859#1859: *1968 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:20:15 [error] 1861#1861: *1980 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:20:18 [error] 1896#1896: *2019 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:20:18 [error] 1897#1897: *2032 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:20:18 [error] 1899#1899: *2044 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:20:20 [error] 1932#1932: *2082 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:20:20 [error] 1933#1933: *2094 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:20:20 [error] 1935#1935: *2108 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:20:22 [error] 1970#1970: *2145 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:20:22 [error] 1971#1971: *2157 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:20:22 [error] 1973#1973: *2171 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:20:24 [error] 2006#2006: *2229 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:20:24 [error] 2007#2007: *2244 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:20:24 [error] 2009#2009: *2258 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:20:26 [error] 2042#2042: *2300 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:20:26 [error] 2043#2043: *2312 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:20:26 [error] 2045#2045: *2326 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:20:28 [error] 2080#2080: *2363 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:20:28 [error] 2081#2081: *2376 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:20:28 [error] 2083#2083: *2390 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:20:30 [error] 2116#2116: *2434 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:20:30 [error] 2117#2117: *2448 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:20:30 [error] 2119#2119: *2462 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:20:33 [error] 2154#2154: *2516 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:20:33 [error] 2155#2155: *2527 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:20:33 [error] 2157#2157: *2540 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:20:33 [error] 2158#2158: *2552 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:20:35 [error] 2190#2190: *2588 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:20:35 [error] 2191#2191: *2600 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:20:36 [error] 2193#2193: *2615 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:20:37 [error] 2233#2233: *2664 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:20:37 [error] 2228#2228: *2679 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:20:37 [error] 2229#2229: *2694 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:20:39 [error] 2267#2267: *2735 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:20:39 [error] 2264#2264: *2747 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:20:39 [error] 2265#2265: *2761 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:20:39 [error] 2268#2268: *2776 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:20:42 [error] 2302#2302: *2812 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:20:42 [error] 2303#2303: *2826 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:20:42 [error] 2305#2305: *2840 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:20:45 [error] 2338#2338: *2898 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:20:45 [error] 2339#2339: *2910 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:20:45 [error] 2341#2341: *2923 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:20:45 [error] 2342#2342: *2937 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:20:48 [error] 2376#2376: *2981 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:20:48 [error] 2377#2377: *2995 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:20:48 [error] 2379#2379: *3008 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:20:48 [error] 2380#2380: *3023 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:20:50 [error] 2413#2413: *3057 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:20:50 [error] 2414#2414: *3071 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:20:51 [error] 2416#2416: *3085 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:20:51 [error] 2417#2417: *3101 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-oss.png b/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-oss.png index e7e2c229f7..d20d674ce2 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-oss.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-oss.png differ diff --git a/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-plus.png b/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-plus.png index ea7ab1486e..29d45a7141 100644 Binary files a/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-plus.png and b/tests/results/scale/edge/TestScale_HTTPSListeners/ttr-plus.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/cpu-oss.png b/tests/results/scale/edge/TestScale_Listeners/cpu-oss.png index ea3db28b65..3b4b588d5c 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/cpu-oss.png and b/tests/results/scale/edge/TestScale_Listeners/cpu-oss.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/cpu-plus.png b/tests/results/scale/edge/TestScale_Listeners/cpu-plus.png index beb015f4c2..9445e1021b 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/cpu-plus.png and b/tests/results/scale/edge/TestScale_Listeners/cpu-plus.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/memory-oss.png b/tests/results/scale/edge/TestScale_Listeners/memory-oss.png index 8d0b0e101c..7b3a341f55 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/memory-oss.png and b/tests/results/scale/edge/TestScale_Listeners/memory-oss.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/memory-plus.png b/tests/results/scale/edge/TestScale_Listeners/memory-plus.png index 856cd9fb5c..79537a24a3 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/memory-plus.png and b/tests/results/scale/edge/TestScale_Listeners/memory-plus.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/ngf-oss.log b/tests/results/scale/edge/TestScale_Listeners/ngf-oss.log index 892742246a..3e0a3a5580 100644 --- a/tests/results/scale/edge/TestScale_Listeners/ngf-oss.log +++ b/tests/results/scale/edge/TestScale_Listeners/ngf-oss.log @@ -1 +1,13 @@ -{"level":"debug","ts":"2025-03-15T18:56:50Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:17:31Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gatewayclasses.gateway.networking.k8s.io \"nginx\": the object has been modified; please apply your changes to the latest version and try again","namespace":"","name":"nginx","kind":"GatewayClass"} +{"level":"debug","ts":"2025-05-20T18:18:12Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:03Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:04Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:04Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:07Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:08Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:15Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:16Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:19Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T18:19:20Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/TestScale_Listeners/ngf-plus.log b/tests/results/scale/edge/TestScale_Listeners/ngf-plus.log index bf1f2e1f65..26829efb1d 100644 --- a/tests/results/scale/edge/TestScale_Listeners/ngf-plus.log +++ b/tests/results/scale/edge/TestScale_Listeners/ngf-plus.log @@ -1,2 +1,9 @@ -{"level":"debug","ts":"2024-11-15T18:20:41Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} -{"level":"debug","ts":"2024-11-15T18:20:42Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:14:11Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:14:57Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:14:59Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:01Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:02Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:08Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:08Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:09Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-05-20T20:15:10Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/TestScale_Listeners/nginx-plus.log b/tests/results/scale/edge/TestScale_Listeners/nginx-plus.log index f063845944..88fffd8dd8 100644 --- a/tests/results/scale/edge/TestScale_Listeners/nginx-plus.log +++ b/tests/results/scale/edge/TestScale_Listeners/nginx-plus.log @@ -1,7 +1,178 @@ -2025/03/15 18:58:32 [error] 127#127: *15 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 0.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-0_80/", host: "0.example.com" -2025/03/15 18:58:43 [error] 774#774: *250 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" -2025/03/15 18:58:50 [error] 1080#1080: *367 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 28.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-28_80/", host: "28.example.com" -2025/03/15 18:58:58 [error] 1318#1318: *462 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 35.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-35_80/", host: "35.example.com" -2025/03/15 18:59:09 [error] 1624#1624: *582 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 44.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-44_80/", host: "44.example.com" -2025/03/15 18:59:16 [error] 1760#1760: *640 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" -2025/03/15 18:59:44 [error] 2270#2270: *874 no live upstreams while connecting to upstream, client: 10.138.0.126, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:14:49 [error] 143#143: *77 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:14:49 [error] 144#144: *78 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:14:49 [error] 146#146: *79 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:14:49 [error] 147#147: *80 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/05/20 20:14:50 [error] 161#161: *93 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:14:50 [error] 162#162: *94 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:14:50 [error] 164#164: *95 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:14:50 [error] 165#165: *96 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/05/20 20:14:51 [error] 179#179: *110 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:14:51 [error] 180#180: *111 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:14:51 [error] 182#182: *112 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:14:51 [error] 183#183: *114 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/05/20 20:14:52 [error] 199#199: *129 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:14:52 [error] 200#200: *130 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:14:52 [error] 202#202: *131 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:14:52 [error] 203#203: *132 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/05/20 20:14:53 [error] 217#217: *148 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:14:53 [error] 218#218: *149 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:14:53 [error] 220#220: *150 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:14:53 [error] 221#221: *151 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/05/20 20:14:54 [error] 235#235: *168 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:14:54 [error] 236#236: *169 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:14:54 [error] 238#238: *170 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:14:54 [error] 239#239: *171 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:14:54 [error] 240#240: *172 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/05/20 20:14:55 [error] 253#253: *190 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:14:55 [error] 254#254: *191 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:14:55 [error] 256#256: *192 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:14:55 [error] 257#257: *193 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:14:55 [error] 258#258: *194 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/05/20 20:14:55 [error] 271#271: *213 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:14:56 [error] 272#272: *214 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:14:56 [error] 274#274: *215 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:14:56 [error] 275#275: *216 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:14:56 [error] 276#276: *217 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/05/20 20:14:56 [error] 291#291: *229 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:14:57 [error] 292#292: *230 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:14:57 [error] 294#294: *231 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:14:57 [error] 295#295: *232 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/05/20 20:14:57 [error] 309#309: *253 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:14:57 [error] 310#310: *254 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:14:58 [error] 312#312: *255 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:14:58 [error] 313#313: *271 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/05/20 20:14:58 [error] 327#327: *293 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:14:58 [error] 328#328: *294 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:14:58 [error] 330#330: *295 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:14:59 [error] 331#331: *296 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/05/20 20:14:59 [error] 345#345: *319 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:14:59 [error] 346#346: *320 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:14:59 [error] 348#348: *321 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:14:59 [error] 349#349: *322 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:15:00 [error] 350#350: *323 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 13.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/05/20 20:15:00 [error] 363#363: *347 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:15:00 [error] 364#364: *348 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:15:00 [error] 366#366: *349 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:15:00 [error] 367#367: *350 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/05/20 20:15:01 [error] 381#381: *375 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:15:01 [error] 382#382: *376 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:15:01 [error] 384#384: *377 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:15:01 [error] 385#385: *378 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:15:02 [error] 386#386: *379 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/05/20 20:15:03 [error] 399#399: *405 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:15:03 [error] 400#400: *406 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:15:03 [error] 402#402: *407 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:15:03 [error] 403#403: *408 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:15:03 [error] 404#404: *409 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/05/20 20:15:04 [error] 417#417: *436 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:15:04 [error] 418#418: *437 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:15:04 [error] 420#420: *438 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:15:04 [error] 421#421: *439 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:15:04 [error] 422#422: *440 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/05/20 20:15:05 [error] 435#435: *468 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:15:05 [error] 436#436: *469 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:15:05 [error] 438#438: *470 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:15:05 [error] 439#439: *471 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:15:05 [error] 440#440: *472 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/05/20 20:15:06 [error] 455#455: *502 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:15:06 [error] 456#456: *503 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:15:06 [error] 458#458: *504 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:15:07 [error] 459#459: *505 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:15:07 [error] 460#460: *506 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/05/20 20:15:09 [error] 527#527: *638 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 23.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-23_80/", host: "23.example.com" +2025/05/20 20:15:10 [error] 545#545: *672 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 24.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-24_80/", host: "24.example.com" +2025/05/20 20:15:11 [error] 563#563: *704 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 25.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-25_80/", host: "25.example.com" +2025/05/20 20:15:12 [error] 583#583: *740 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 26.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-26_80/", host: "26.example.com" +2025/05/20 20:15:14 [error] 619#619: *812 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 28.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-28_80/", host: "28.example.com" +2025/05/20 20:15:15 [error] 655#655: *889 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 30.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-30_80/", host: "30.example.com" +2025/05/20 20:15:17 [error] 675#675: *928 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 31.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-31_80/", host: "31.example.com" +2025/05/20 20:15:18 [error] 693#693: *980 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 32.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-32_80/", host: "32.example.com" +2025/05/20 20:15:19 [error] 711#711: *1019 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 33.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/05/20 20:15:19 [error] 712#712: *1032 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 33.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/05/20 20:15:20 [error] 729#729: *1065 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 34.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-34_80/", host: "34.example.com" +2025/05/20 20:15:21 [error] 747#747: *1107 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 35.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/05/20 20:15:21 [error] 748#748: *1120 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 35.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/05/20 20:15:22 [error] 767#767: *1154 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 36.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/05/20 20:15:22 [error] 768#768: *1167 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 36.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/05/20 20:15:23 [error] 785#785: *1202 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 37.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/05/20 20:15:24 [error] 786#786: *1213 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 37.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/05/20 20:15:25 [error] 803#803: *1248 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 38.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/05/20 20:15:25 [error] 804#804: *1261 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 38.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/05/20 20:15:26 [error] 823#823: *1297 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 39.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/05/20 20:15:27 [error] 824#824: *1310 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 39.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/05/20 20:15:28 [error] 841#841: *1363 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 40.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/05/20 20:15:28 [error] 842#842: *1375 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 40.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/05/20 20:15:29 [error] 859#859: *1407 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 41.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/05/20 20:15:29 [error] 860#860: *1420 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 41.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/05/20 20:15:29 [error] 862#862: *1432 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 41.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/05/20 20:15:31 [error] 879#879: *1467 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 42.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/05/20 20:15:31 [error] 880#880: *1479 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 42.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/05/20 20:15:33 [error] 897#897: *1518 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 43.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/05/20 20:15:33 [error] 898#898: *1531 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 43.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/05/20 20:15:35 [error] 915#915: *1571 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 44.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/05/20 20:15:35 [error] 916#916: *1583 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 44.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/05/20 20:15:36 [error] 933#933: *1624 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/05/20 20:15:36 [error] 934#934: *1636 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/05/20 20:15:36 [error] 936#936: *1649 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/05/20 20:15:38 [error] 951#951: *1692 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 46.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/05/20 20:15:38 [error] 952#952: *1704 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 46.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/05/20 20:15:38 [error] 954#954: *1717 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 46.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/05/20 20:15:39 [error] 969#969: *1749 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:15:40 [error] 970#970: *1762 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:15:40 [error] 972#972: *1775 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/05/20 20:15:41 [error] 989#989: *1810 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:15:42 [error] 990#990: *1822 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:15:42 [error] 992#992: *1835 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/05/20 20:15:43 [error] 1007#1007: *1875 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:15:43 [error] 1008#1008: *1885 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:15:43 [error] 1010#1010: *1896 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/05/20 20:15:44 [error] 1025#1025: *1928 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:15:44 [error] 1026#1026: *1942 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:15:44 [error] 1028#1028: *1955 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/05/20 20:15:46 [error] 1043#1043: *1992 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:15:46 [error] 1044#1044: *2004 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:15:46 [error] 1046#1046: *2017 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/05/20 20:15:48 [error] 1063#1063: *2069 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:15:49 [error] 1064#1064: *2081 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:15:49 [error] 1066#1066: *2093 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 52.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/05/20 20:15:50 [error] 1081#1081: *2132 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:15:50 [error] 1082#1082: *2146 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:15:51 [error] 1084#1084: *2159 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/05/20 20:15:52 [error] 1101#1101: *2194 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:15:52 [error] 1102#1102: *2207 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:15:53 [error] 1104#1104: *2220 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:15:53 [error] 1105#1105: *2233 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/05/20 20:15:54 [error] 1119#1119: *2262 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:15:54 [error] 1120#1120: *2276 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:15:54 [error] 1122#1122: *2290 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/05/20 20:15:56 [error] 1139#1139: *2334 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:15:56 [error] 1140#1140: *2346 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:15:57 [error] 1142#1142: *2359 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/05/20 20:15:59 [error] 1157#1157: *2420 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:15:59 [error] 1158#1158: *2433 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:15:59 [error] 1160#1160: *2446 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/05/20 20:16:01 [error] 1175#1175: *2477 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:16:01 [error] 1176#1176: *2490 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:16:01 [error] 1178#1178: *2502 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:16:01 [error] 1179#1179: *2515 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/05/20 20:16:04 [error] 1195#1195: *2550 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:16:04 [error] 1196#1196: *2562 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:16:04 [error] 1198#1198: *2575 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:16:04 [error] 1199#1199: *2588 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/05/20 20:16:05 [error] 1213#1213: *2622 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:16:05 [error] 1214#1214: *2636 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:16:05 [error] 1216#1216: *2649 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:16:05 [error] 1217#1217: *2663 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/05/20 20:16:07 [error] 1233#1233: *2691 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:16:07 [error] 1234#1234: *2703 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:16:07 [error] 1236#1236: *2716 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:16:07 [error] 1237#1237: *2727 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:16:07 [error] 1238#1238: *2739 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/05/20 20:16:09 [error] 1251#1251: *2784 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:16:09 [error] 1252#1252: *2797 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:16:09 [error] 1254#1254: *2810 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:16:09 [error] 1255#1255: *2824 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/05/20 20:16:11 [error] 1271#1271: *2856 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:16:12 [error] 1272#1272: *2869 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:16:12 [error] 1274#1274: *2882 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/05/20 20:16:12 [error] 1275#1275: *2895 no live upstreams while connecting to upstream, client: 34.83.228.29, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" diff --git a/tests/results/scale/edge/TestScale_Listeners/ttr-oss.png b/tests/results/scale/edge/TestScale_Listeners/ttr-oss.png index 390469af6f..888b2a1bf8 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/ttr-oss.png and b/tests/results/scale/edge/TestScale_Listeners/ttr-oss.png differ diff --git a/tests/results/scale/edge/TestScale_Listeners/ttr-plus.png b/tests/results/scale/edge/TestScale_Listeners/ttr-plus.png index f41386ec7a..520d034bf5 100644 Binary files a/tests/results/scale/edge/TestScale_Listeners/ttr-plus.png and b/tests/results/scale/edge/TestScale_Listeners/ttr-plus.png differ diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/cpu-oss.png b/tests/results/scale/edge/TestScale_UpstreamServers/cpu-oss.png index 0899136b25..c25d9ed212 100644 Binary files a/tests/results/scale/edge/TestScale_UpstreamServers/cpu-oss.png and b/tests/results/scale/edge/TestScale_UpstreamServers/cpu-oss.png differ diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/cpu-plus.png b/tests/results/scale/edge/TestScale_UpstreamServers/cpu-plus.png index b9672a72b3..f7e74a90a5 100644 Binary files a/tests/results/scale/edge/TestScale_UpstreamServers/cpu-plus.png and b/tests/results/scale/edge/TestScale_UpstreamServers/cpu-plus.png differ diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/memory-oss.png b/tests/results/scale/edge/TestScale_UpstreamServers/memory-oss.png index bb06b99b8d..4efbcba315 100644 Binary files a/tests/results/scale/edge/TestScale_UpstreamServers/memory-oss.png and b/tests/results/scale/edge/TestScale_UpstreamServers/memory-oss.png differ diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/memory-plus.png b/tests/results/scale/edge/TestScale_UpstreamServers/memory-plus.png index 26cdc8ce2d..7d59a00e20 100644 Binary files a/tests/results/scale/edge/TestScale_UpstreamServers/memory-plus.png and b/tests/results/scale/edge/TestScale_UpstreamServers/memory-plus.png differ diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/ngf-oss.log b/tests/results/scale/edge/TestScale_UpstreamServers/ngf-oss.log new file mode 100644 index 0000000000..eb36a6bf5a --- /dev/null +++ b/tests/results/scale/edge/TestScale_UpstreamServers/ngf-oss.log @@ -0,0 +1 @@ +{"level":"debug","ts":"2025-05-20T18:34:39Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/TestScale_UpstreamServers/ngf-plus.log b/tests/results/scale/edge/TestScale_UpstreamServers/ngf-plus.log index 35862e1388..ea9fba15cf 100644 --- a/tests/results/scale/edge/TestScale_UpstreamServers/ngf-plus.log +++ b/tests/results/scale/edge/TestScale_UpstreamServers/ngf-plus.log @@ -1 +1 @@ -{"level":"info","ts":"2024-10-01T16:34:56Z","msg":"pkg/mod/k8s.io/client-go@v0.31.1/tools/cache/reflector.go:243: watch of *v1.EndpointSlice ended with: an error on the server (\"unable to decode an event from the watch stream: got short buffer with n=0, base=4092, cap=81920\") has prevented the request from succeeding"} +{"level":"debug","ts":"2025-05-20T20:33:30Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/edge/edge-oss.md b/tests/results/scale/edge/edge-oss.md index e09c77e77a..0ef082509d 100644 --- a/tests/results/scale/edge/edge-oss.md +++ b/tests/results/scale/edge/edge-oss.md @@ -6,50 +6,37 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851332Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 ## Test TestScale_Listeners -### Reloads - -- Total: 127 -- Total Errors: 0 -- Average Time: 127ms -- Reload distribution: - - 500.0ms: 127 - - 1000.0ms: 127 - - 5000.0ms: 127 - - 10000.0ms: 127 - - 30000.0ms: 127 - - +Infms: 127 - ### Event Batch Processing -- Total: 385 -- Average Time: 136ms +- Total: 325 +- Average Time: 6ms - Event Batch Processing distribution: - - 500.0ms: 347 - - 1000.0ms: 382 - - 5000.0ms: 385 - - 10000.0ms: 385 - - 30000.0ms: 385 - - +Infms: 385 + - 500.0ms: 325 + - 1000.0ms: 325 + - 5000.0ms: 325 + - 10000.0ms: 325 + - 30000.0ms: 325 + - +Infms: 325 ### Errors -- NGF errors: 1 +- NGF errors: 13 - NGF container restarts: 0 - NGINX errors: 0 - NGINX container restarts: 0 @@ -61,34 +48,21 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPSListeners -### Reloads - -- Total: 128 -- Total Errors: 0 -- Average Time: 146ms -- Reload distribution: - - 500.0ms: 128 - - 1000.0ms: 128 - - 5000.0ms: 128 - - 10000.0ms: 128 - - 30000.0ms: 128 - - +Infms: 128 - ### Event Batch Processing -- Total: 450 -- Average Time: 166ms +- Total: 392 +- Average Time: 18ms - Event Batch Processing distribution: - 500.0ms: 392 - - 1000.0ms: 432 - - 5000.0ms: 450 - - 10000.0ms: 450 - - 30000.0ms: 450 - - +Infms: 450 + - 1000.0ms: 392 + - 5000.0ms: 392 + - 10000.0ms: 392 + - 30000.0ms: 392 + - +Infms: 392 ### Errors -- NGF errors: 0 +- NGF errors: 18 - NGF container restarts: 0 - NGINX errors: 0 - NGINX container restarts: 0 @@ -100,30 +74,17 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPRoutes -### Reloads - -- Total: 1001 -- Total Errors: 0 -- Average Time: 174ms -- Reload distribution: - - 500.0ms: 1001 - - 1000.0ms: 1001 - - 5000.0ms: 1001 - - 10000.0ms: 1001 - - 30000.0ms: 1001 - - +Infms: 1001 - ### Event Batch Processing -- Total: 1008 -- Average Time: 229ms +- Total: 1009 +- Average Time: 92ms - Event Batch Processing distribution: - - 500.0ms: 1002 - - 1000.0ms: 1008 - - 5000.0ms: 1008 - - 10000.0ms: 1008 - - 30000.0ms: 1008 - - +Infms: 1008 + - 500.0ms: 1009 + - 1000.0ms: 1009 + - 5000.0ms: 1009 + - 10000.0ms: 1009 + - 30000.0ms: 1009 + - +Infms: 1009 ### Errors @@ -139,34 +100,21 @@ The logs are attached only if there are errors. ## Test TestScale_UpstreamServers -### Reloads - -- Total: 97 -- Total Errors: 0 -- Average Time: 126ms -- Reload distribution: - - 500.0ms: 97 - - 1000.0ms: 97 - - 5000.0ms: 97 - - 10000.0ms: 97 - - 30000.0ms: 97 - - +Infms: 97 - ### Event Batch Processing -- Total: 99 -- Average Time: 125ms +- Total: 157 +- Average Time: 76ms - Event Batch Processing distribution: - - 500.0ms: 99 - - 1000.0ms: 99 - - 5000.0ms: 99 - - 10000.0ms: 99 - - 30000.0ms: 99 - - +Infms: 99 + - 500.0ms: 157 + - 1000.0ms: 157 + - 5000.0ms: 157 + - 10000.0ms: 157 + - 30000.0ms: 157 + - +Infms: 157 ### Errors -- NGF errors: 0 +- NGF errors: 1 - NGF container restarts: 0 - NGINX errors: 0 - NGINX container restarts: 0 @@ -179,20 +127,20 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPMatches ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.01 -Duration [total, attack, wait] 30s, 29.999s, 663.238µs -Latencies [min, mean, 50, 90, 95, 99, max] 499.976µs, 677.946µs, 660.823µs, 759.984µs, 799.116µs, 904.939µs, 12.162ms -Bytes In [total, mean] 4830000, 161.00 +Requests [total, rate, throughput] 30000, 1000.04, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 988.104µs +Latencies [min, mean, 50, 90, 95, 99, max] 712.191µs, 939.08µs, 919.118µs, 1.034ms, 1.081ms, 1.213ms, 22.584ms +Bytes In [total, mean] 4770000, 159.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 Error Set: ``` ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 750.337µs -Latencies [min, mean, 50, 90, 95, 99, max] 590.522µs, 762.674µs, 740.085µs, 869.449µs, 930.564µs, 1.057ms, 8.287ms -Bytes In [total, mean] 4830000, 161.00 +Requests [total, rate, throughput] 30000, 1000.04, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 987.954µs +Latencies [min, mean, 50, 90, 95, 99, max] 819.222µs, 1.026ms, 1.008ms, 1.126ms, 1.178ms, 1.327ms, 10.006ms +Bytes In [total, mean] 4770000, 159.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 diff --git a/tests/results/scale/edge/edge-plus.md b/tests/results/scale/edge/edge-plus.md index 42bff5367d..de993dd546 100644 --- a/tests/results/scale/edge/edge-plus.md +++ b/tests/results/scale/edge/edge-plus.md @@ -6,14 +6,14 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 - RAM per node: 65851340Ki - Max pods per node: 110 @@ -22,36 +22,23 @@ GKE Cluster: ## Test TestScale_Listeners -### Reloads - -- Total: 128 -- Total Errors: 0 -- Average Time: 151ms -- Reload distribution: - - 500.0ms: 128 - - 1000.0ms: 128 - - 5000.0ms: 128 - - 10000.0ms: 128 - - 30000.0ms: 128 - - +Infms: 128 - ### Event Batch Processing -- Total: 387 -- Average Time: 134ms +- Total: 325 +- Average Time: 128ms - Event Batch Processing distribution: - - 500.0ms: 351 - - 1000.0ms: 386 - - 5000.0ms: 387 - - 10000.0ms: 387 - - 30000.0ms: 387 - - +Infms: 387 + - 500.0ms: 283 + - 1000.0ms: 325 + - 5000.0ms: 325 + - 10000.0ms: 325 + - 30000.0ms: 325 + - +Infms: 325 ### Errors -- NGF errors: 0 +- NGF errors: 9 - NGF container restarts: 0 -- NGINX errors: 7 +- NGINX errors: 178 - NGINX container restarts: 0 ### Graphs and Logs @@ -61,36 +48,23 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPSListeners -### Reloads - -- Total: 128 -- Total Errors: 0 -- Average Time: 160ms -- Reload distribution: - - 500.0ms: 128 - - 1000.0ms: 128 - - 5000.0ms: 128 - - 10000.0ms: 128 - - 30000.0ms: 128 - - +Infms: 128 - ### Event Batch Processing -- Total: 451 -- Average Time: 127ms +- Total: 390 +- Average Time: 120ms - Event Batch Processing distribution: - - 500.0ms: 408 - - 1000.0ms: 450 - - 5000.0ms: 451 - - 10000.0ms: 451 - - 30000.0ms: 451 - - +Infms: 451 + - 500.0ms: 345 + - 1000.0ms: 390 + - 5000.0ms: 390 + - 10000.0ms: 390 + - 30000.0ms: 390 + - +Infms: 390 ### Errors -- NGF errors: 0 +- NGF errors: 14 - NGF container restarts: 0 -- NGINX errors: 15 +- NGINX errors: 171 - NGINX container restarts: 0 ### Graphs and Logs @@ -100,30 +74,17 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPRoutes -### Reloads - -- Total: 1001 -- Total Errors: 0 -- Average Time: 189ms -- Reload distribution: - - 500.0ms: 1001 - - 1000.0ms: 1001 - - 5000.0ms: 1001 - - 10000.0ms: 1001 - - 30000.0ms: 1001 - - +Infms: 1001 - ### Event Batch Processing -- Total: 1008 -- Average Time: 261ms +- Total: 1009 +- Average Time: 171ms - Event Batch Processing distribution: - - 500.0ms: 1006 - - 1000.0ms: 1008 - - 5000.0ms: 1008 - - 10000.0ms: 1008 - - 30000.0ms: 1008 - - +Infms: 1008 + - 500.0ms: 1008 + - 1000.0ms: 1009 + - 5000.0ms: 1009 + - 10000.0ms: 1009 + - 30000.0ms: 1009 + - +Infms: 1009 ### Errors @@ -139,34 +100,21 @@ The logs are attached only if there are errors. ## Test TestScale_UpstreamServers -### Reloads - -- Total: 3 -- Total Errors: 0 -- Average Time: 143ms -- Reload distribution: - - 500.0ms: 3 - - 1000.0ms: 3 - - 5000.0ms: 3 - - 10000.0ms: 3 - - 30000.0ms: 3 - - +Infms: 3 - ### Event Batch Processing -- Total: 37 -- Average Time: 498ms +- Total: 33 +- Average Time: 511ms - Event Batch Processing distribution: - 500.0ms: 19 - - 1000.0ms: 35 - - 5000.0ms: 37 - - 10000.0ms: 37 - - 30000.0ms: 37 - - +Infms: 37 + - 1000.0ms: 29 + - 5000.0ms: 33 + - 10000.0ms: 33 + - 30000.0ms: 33 + - +Infms: 33 ### Errors -- NGF errors: 0 +- NGF errors: 1 - NGF container restarts: 0 - NGINX errors: 0 - NGINX container restarts: 0 @@ -179,20 +127,20 @@ The logs are attached only if there are errors. ## Test TestScale_HTTPMatches ```text -Requests [total, rate, throughput] 30000, 1000.02, 1000.00 -Duration [total, attack, wait] 30s, 29.999s, 666.245µs -Latencies [min, mean, 50, 90, 95, 99, max] 514.253µs, 675.464µs, 655.764µs, 737.887µs, 766.943µs, 852.013µs, 12.375ms -Bytes In [total, mean] 4860000, 162.00 +Requests [total, rate, throughput] 29999, 1000.00, 999.98 +Duration [total, attack, wait] 30s, 29.999s, 800.545µs +Latencies [min, mean, 50, 90, 95, 99, max] 691.408µs, 928.268µs, 907.578µs, 1.031ms, 1.076ms, 1.197ms, 15.712ms +Bytes In [total, mean] 4799840, 160.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:30000 +Status Codes [code:count] 200:29999 Error Set: ``` ```text -Requests [total, rate, throughput] 30000, 1000.03, 1000.01 -Duration [total, attack, wait] 30s, 29.999s, 772.346µs -Latencies [min, mean, 50, 90, 95, 99, max] 596.801µs, 753.715µs, 734.197µs, 841.051µs, 886.584µs, 980.974µs, 13.362ms -Bytes In [total, mean] 4860000, 162.00 +Requests [total, rate, throughput] 30000, 1000.02, 999.99 +Duration [total, attack, wait] 30s, 29.999s, 941.298µs +Latencies [min, mean, 50, 90, 95, 99, max] 789.486µs, 1.009ms, 991.314µs, 1.118ms, 1.175ms, 1.301ms, 22.133ms +Bytes In [total, mean] 4800000, 160.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-oss.png index c6629a529a..00a68822ce 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-plus.png index c262e56fa0..908b718bf9 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-oss.png index c6629a529a..00a68822ce 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-plus.png index c262e56fa0..2cdf756754 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-oss.png index 33783f67af..9fe617cf97 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-plus.png index b436e0cb38..e7547c6a1f 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-oss.png index 33783f67af..9fe617cf97 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-plus.png index b436e0cb38..e7547c6a1f 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-down-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-oss.png index 5197b18ead..0e07dac135 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-plus.png index f83464b470..7272f71f7d 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-oss.png index 5197b18ead..0e07dac135 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-plus.png index f83464b470..451c709430 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-oss.png index 6ed5b65e55..5a86ddd038 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-plus.png index b842efc1d4..fde3d5e90d 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-oss.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-oss.png index 6ed5b65e55..5a86ddd038 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-oss.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-plus.png b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-plus.png index b842efc1d4..a5871c868c 100644 Binary files a/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-plus.png and b/tests/results/zero-downtime-scale/edge/abrupt-scale-up-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/edge-oss.md b/tests/results/zero-downtime-scale/edge/edge-oss.md index ac4d398884..0838cba77d 100644 --- a/tests/results/zero-downtime-scale/edge/edge-oss.md +++ b/tests/results/zero-downtime-scale/edge/edge-oss.md @@ -6,21 +6,21 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851332Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 -## One NGF Pod runs per node Test Results +## One NGINX Pod runs per node Test Results ### Scale Up Gradually @@ -28,9 +28,9 @@ GKE Cluster: ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 806.452µs -Latencies [min, mean, 50, 90, 95, 99, max] 433.68µs, 873.614µs, 867.783µs, 993.691µs, 1.043ms, 1.351ms, 12.859ms -Bytes In [total, mean] 4646890, 154.90 +Duration [total, attack, wait] 5m0s, 5m0s, 1.079ms +Latencies [min, mean, 50, 90, 95, 99, max] 656.562µs, 1.243ms, 1.194ms, 1.385ms, 1.475ms, 1.856ms, 218.579ms +Bytes In [total, mean] 4595884, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 @@ -43,9 +43,9 @@ Error Set: ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 1.573ms -Latencies [min, mean, 50, 90, 95, 99, max] 412.342µs, 847.173µs, 847.018µs, 972.99µs, 1.017ms, 1.314ms, 12.59ms -Bytes In [total, mean] 4823979, 160.80 +Duration [total, attack, wait] 5m0s, 5m0s, 1.11ms +Latencies [min, mean, 50, 90, 95, 99, max] 660.196µs, 1.214ms, 1.182ms, 1.366ms, 1.446ms, 1.83ms, 209.558ms +Bytes In [total, mean] 4776017, 159.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 @@ -56,35 +56,35 @@ Error Set: ### Scale Down Gradually -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 48000, 100.00, 100.00 -Duration [total, attack, wait] 8m0s, 8m0s, 901.369µs -Latencies [min, mean, 50, 90, 95, 99, max] 400.666µs, 872.136µs, 870.908µs, 1.012ms, 1.064ms, 1.269ms, 12.942ms -Bytes In [total, mean] 7718342, 160.80 +Duration [total, attack, wait] 8m0s, 8m0s, 1.151ms +Latencies [min, mean, 50, 90, 95, 99, max] 651.444µs, 1.194ms, 1.181ms, 1.348ms, 1.417ms, 1.747ms, 31.936ms +Bytes In [total, mean] 7353552, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:48000 Error Set: ``` -![gradual-scale-down-affinity-http-oss.png](gradual-scale-down-affinity-http-oss.png) +![gradual-scale-down-affinity-https-oss.png](gradual-scale-down-affinity-https-oss.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 48000, 100.00, 100.00 -Duration [total, attack, wait] 8m0s, 8m0s, 941.614µs -Latencies [min, mean, 50, 90, 95, 99, max] 417.468µs, 901.578µs, 895.785µs, 1.037ms, 1.091ms, 1.325ms, 13.219ms -Bytes In [total, mean] 7435362, 154.90 +Duration [total, attack, wait] 8m0s, 8m0s, 1.543ms +Latencies [min, mean, 50, 90, 95, 99, max] 625.439µs, 1.158ms, 1.149ms, 1.315ms, 1.378ms, 1.697ms, 29.523ms +Bytes In [total, mean] 7641540, 159.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:48000 Error Set: ``` -![gradual-scale-down-affinity-https-oss.png](gradual-scale-down-affinity-https-oss.png) +![gradual-scale-down-affinity-http-oss.png](gradual-scale-down-affinity-http-oss.png) ### Scale Up Abruptly @@ -92,9 +92,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 1.117ms -Latencies [min, mean, 50, 90, 95, 99, max] 455.668µs, 904.422µs, 904.957µs, 1.033ms, 1.082ms, 1.248ms, 5.631ms -Bytes In [total, mean] 1858693, 154.89 +Duration [total, attack, wait] 2m0s, 2m0s, 1.212ms +Latencies [min, mean, 50, 90, 95, 99, max] 648.427µs, 1.189ms, 1.186ms, 1.343ms, 1.398ms, 1.591ms, 8.864ms +Bytes In [total, mean] 1838463, 153.21 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 @@ -107,9 +107,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 1.119ms -Latencies [min, mean, 50, 90, 95, 99, max] 412.944µs, 891.737µs, 891.196µs, 1.028ms, 1.081ms, 1.282ms, 9.702ms -Bytes In [total, mean] 1929586, 160.80 +Duration [total, attack, wait] 2m0s, 2m0s, 1.033ms +Latencies [min, mean, 50, 90, 95, 99, max] 683.202µs, 1.25ms, 1.191ms, 1.375ms, 1.442ms, 1.687ms, 215.314ms +Bytes In [total, mean] 1910511, 159.21 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 @@ -124,9 +124,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 713.199µs -Latencies [min, mean, 50, 90, 95, 99, max] 494.231µs, 953.041µs, 942.581µs, 1.108ms, 1.176ms, 1.337ms, 10.627ms -Bytes In [total, mean] 1858836, 154.90 +Duration [total, attack, wait] 2m0s, 2m0s, 1.186ms +Latencies [min, mean, 50, 90, 95, 99, max] 685.26µs, 1.191ms, 1.188ms, 1.346ms, 1.409ms, 1.594ms, 12.06ms +Bytes In [total, mean] 1838451, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 @@ -139,9 +139,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 968.062µs -Latencies [min, mean, 50, 90, 95, 99, max] 478.648µs, 936.553µs, 928.014µs, 1.105ms, 1.171ms, 1.332ms, 10.648ms -Bytes In [total, mean] 1929503, 160.79 +Duration [total, attack, wait] 2m0s, 2m0s, 823.136µs +Latencies [min, mean, 50, 90, 95, 99, max] 646.596µs, 1.162ms, 1.166ms, 1.325ms, 1.381ms, 1.562ms, 12.055ms +Bytes In [total, mean] 1910395, 159.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 @@ -150,103 +150,104 @@ Error Set: ![abrupt-scale-down-affinity-http-oss.png](abrupt-scale-down-affinity-http-oss.png) -## Multiple NGF Pods run per node Test Results +## Multiple NGINX Pods run per node Test Results ### Scale Up Gradually -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 873.094µs -Latencies [min, mean, 50, 90, 95, 99, max] 443.965µs, 917.539µs, 914.468µs, 1.045ms, 1.096ms, 1.453ms, 25.238ms -Bytes In [total, mean] 4829927, 161.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.174ms +Latencies [min, mean, 50, 90, 95, 99, max] 639.841µs, 1.198ms, 1.182ms, 1.353ms, 1.43ms, 1.818ms, 24.143ms +Bytes In [total, mean] 4619796, 153.99 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:30000 +Status Codes [code:count] 0:1 200:29999 Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->35.230.31.19:443: connect: connection refused ``` -![gradual-scale-up-http-oss.png](gradual-scale-up-http-oss.png) +![gradual-scale-up-https-oss.png](gradual-scale-up-https-oss.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 860.394µs -Latencies [min, mean, 50, 90, 95, 99, max] 468.09µs, 940.797µs, 930.888µs, 1.071ms, 1.133ms, 1.485ms, 25.057ms -Bytes In [total, mean] 4652973, 155.10 +Duration [total, attack, wait] 5m0s, 5m0s, 977.684µs +Latencies [min, mean, 50, 90, 95, 99, max] 632.63µs, 1.221ms, 1.168ms, 1.342ms, 1.418ms, 1.805ms, 215.589ms +Bytes In [total, mean] 4785071, 159.50 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 Error Set: ``` -![gradual-scale-up-https-oss.png](gradual-scale-up-https-oss.png) +![gradual-scale-up-http-oss.png](gradual-scale-up-http-oss.png) ### Scale Down Gradually -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 96000, 100.00, 100.00 -Duration [total, attack, wait] 16m0s, 16m0s, 1.096ms -Latencies [min, mean, 50, 90, 95, 99, max] 470.107µs, 956.329µs, 946.839µs, 1.102ms, 1.167ms, 1.419ms, 22.663ms -Bytes In [total, mean] 14889791, 155.10 +Duration [total, attack, wait] 16m0s, 16m0s, 1.227ms +Latencies [min, mean, 50, 90, 95, 99, max] 586.914µs, 1.194ms, 1.18ms, 1.347ms, 1.417ms, 1.749ms, 217.951ms +Bytes In [total, mean] 15312219, 159.50 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:96000 Error Set: ``` -![gradual-scale-down-https-oss.png](gradual-scale-down-https-oss.png) +![gradual-scale-down-http-oss.png](gradual-scale-down-http-oss.png) -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 96000, 100.00, 100.00 -Duration [total, attack, wait] 16m0s, 16m0s, 909.045µs -Latencies [min, mean, 50, 90, 95, 99, max] 438.459µs, 925.315µs, 923.188µs, 1.065ms, 1.12ms, 1.373ms, 21.282ms -Bytes In [total, mean] 15456075, 161.00 +Duration [total, attack, wait] 16m0s, 16m0s, 1.215ms +Latencies [min, mean, 50, 90, 95, 99, max] 642.507µs, 1.213ms, 1.195ms, 1.368ms, 1.443ms, 1.778ms, 209.657ms +Bytes In [total, mean] 14783854, 154.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:96000 Error Set: ``` -![gradual-scale-down-http-oss.png](gradual-scale-down-http-oss.png) +![gradual-scale-down-https-oss.png](gradual-scale-down-https-oss.png) ### Scale Up Abruptly -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 1.1ms -Latencies [min, mean, 50, 90, 95, 99, max] 534.726µs, 973.229µs, 963.738µs, 1.127ms, 1.189ms, 1.369ms, 7.262ms -Bytes In [total, mean] 1861155, 155.10 +Duration [total, attack, wait] 2m0s, 2m0s, 1.17ms +Latencies [min, mean, 50, 90, 95, 99, max] 687.211µs, 1.228ms, 1.216ms, 1.398ms, 1.471ms, 1.799ms, 34.351ms +Bytes In [total, mean] 1913915, 159.49 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-up-https-oss.png](abrupt-scale-up-https-oss.png) +![abrupt-scale-up-http-oss.png](abrupt-scale-up-http-oss.png) -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 912.394µs -Latencies [min, mean, 50, 90, 95, 99, max] 484.225µs, 933.174µs, 935.072µs, 1.065ms, 1.109ms, 1.305ms, 11.145ms -Bytes In [total, mean] 1932089, 161.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.111ms +Latencies [min, mean, 50, 90, 95, 99, max] 732.378µs, 1.272ms, 1.251ms, 1.465ms, 1.547ms, 1.839ms, 21.969ms +Bytes In [total, mean] 1847974, 154.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-up-http-oss.png](abrupt-scale-up-http-oss.png) +![abrupt-scale-up-https-oss.png](abrupt-scale-up-https-oss.png) ### Scale Down Abruptly @@ -254,9 +255,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 844.643µs -Latencies [min, mean, 50, 90, 95, 99, max] 452.528µs, 904.647µs, 912.865µs, 1.037ms, 1.076ms, 1.194ms, 6.917ms -Bytes In [total, mean] 1932017, 161.00 +Duration [total, attack, wait] 2m0s, 2m0s, 1.305ms +Latencies [min, mean, 50, 90, 95, 99, max] 690.951µs, 1.257ms, 1.238ms, 1.459ms, 1.559ms, 1.893ms, 13.163ms +Bytes In [total, mean] 1913879, 159.49 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 @@ -269,9 +270,9 @@ Error Set: ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 701.654µs -Latencies [min, mean, 50, 90, 95, 99, max] 511.224µs, 936.573µs, 932.311µs, 1.074ms, 1.128ms, 1.304ms, 7.938ms -Bytes In [total, mean] 1861196, 155.10 +Duration [total, attack, wait] 2m0s, 2m0s, 1.312ms +Latencies [min, mean, 50, 90, 95, 99, max] 728.606µs, 1.306ms, 1.267ms, 1.498ms, 1.602ms, 1.983ms, 37.944ms +Bytes In [total, mean] 1847937, 153.99 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 diff --git a/tests/results/zero-downtime-scale/edge/edge-plus.md b/tests/results/zero-downtime-scale/edge/edge-plus.md index d689897ffb..72f35717eb 100644 --- a/tests/results/zero-downtime-scale/edge/edge-plus.md +++ b/tests/results/zero-downtime-scale/edge/edge-plus.md @@ -6,53 +6,53 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: 66d6f36a8896cb0991348a7acc380ff4897a7e96 +- Date: 2025-05-20T17:14:57Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.32.3-gke.1785003 - vCPUs per node: 16 - RAM per node: 65851340Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 -## One NGF Pod runs per node Test Results +## One NGINX Pod runs per node Test Results ### Scale Up Gradually -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 766.303µs -Latencies [min, mean, 50, 90, 95, 99, max] 441.594µs, 875.579µs, 868.868µs, 997.175µs, 1.049ms, 1.357ms, 13.238ms -Bytes In [total, mean] 4673932, 155.80 +Duration [total, attack, wait] 5m0s, 5m0s, 1.281ms +Latencies [min, mean, 50, 90, 95, 99, max] 515.409µs, 1.099ms, 1.08ms, 1.262ms, 1.33ms, 1.614ms, 218.516ms +Bytes In [total, mean] 4775988, 159.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 Error Set: ``` -![gradual-scale-up-affinity-https-plus.png](gradual-scale-up-affinity-https-plus.png) +![gradual-scale-up-affinity-http-plus.png](gradual-scale-up-affinity-http-plus.png) -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 727.187µs -Latencies [min, mean, 50, 90, 95, 99, max] 414.641µs, 846.924µs, 846.028µs, 971.491µs, 1.017ms, 1.294ms, 11.941ms -Bytes In [total, mean] 4850987, 161.70 +Duration [total, attack, wait] 5m0s, 5m0s, 1.256ms +Latencies [min, mean, 50, 90, 95, 99, max] 589.919µs, 1.164ms, 1.113ms, 1.323ms, 1.404ms, 1.692ms, 216.981ms +Bytes In [total, mean] 4596070, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 Error Set: ``` -![gradual-scale-up-affinity-http-plus.png](gradual-scale-up-affinity-http-plus.png) +![gradual-scale-up-affinity-https-plus.png](gradual-scale-up-affinity-https-plus.png) ### Scale Down Gradually @@ -60,9 +60,9 @@ Error Set: ```text Requests [total, rate, throughput] 48000, 100.00, 100.00 -Duration [total, attack, wait] 8m0s, 8m0s, 730.887µs -Latencies [min, mean, 50, 90, 95, 99, max] 433.836µs, 850.845µs, 848.555µs, 968.862µs, 1.013ms, 1.215ms, 8.39ms -Bytes In [total, mean] 7478267, 155.80 +Duration [total, attack, wait] 8m0s, 8m0s, 1.532ms +Latencies [min, mean, 50, 90, 95, 99, max] 637.028µs, 1.163ms, 1.157ms, 1.335ms, 1.4ms, 1.615ms, 20.507ms +Bytes In [total, mean] 7353627, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:48000 @@ -75,9 +75,9 @@ Error Set: ```text Requests [total, rate, throughput] 48000, 100.00, 100.00 -Duration [total, attack, wait] 8m0s, 8m0s, 715.71µs -Latencies [min, mean, 50, 90, 95, 99, max] 405.345µs, 820.868µs, 825.255µs, 941.274µs, 982.586µs, 1.188ms, 11.166ms -Bytes In [total, mean] 7761660, 161.70 +Duration [total, attack, wait] 8m0s, 8m0s, 1.558ms +Latencies [min, mean, 50, 90, 95, 99, max] 562.998µs, 1.131ms, 1.128ms, 1.304ms, 1.361ms, 1.566ms, 40.018ms +Bytes In [total, mean] 7641585, 159.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:48000 @@ -88,115 +88,120 @@ Error Set: ### Scale Up Abruptly -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 731.03µs -Latencies [min, mean, 50, 90, 95, 99, max] 423.637µs, 822.777µs, 824.747µs, 937.12µs, 981.297µs, 1.14ms, 3.973ms -Bytes In [total, mean] 1940496, 161.71 +Duration [total, attack, wait] 2m0s, 2m0s, 1.272ms +Latencies [min, mean, 50, 90, 95, 99, max] 634.091µs, 1.178ms, 1.138ms, 1.301ms, 1.36ms, 1.573ms, 208.54ms +Bytes In [total, mean] 1838488, 153.21 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-up-affinity-http-plus.png](abrupt-scale-up-affinity-http-plus.png) +![abrupt-scale-up-affinity-https-plus.png](abrupt-scale-up-affinity-https-plus.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text -Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 887.292µs -Latencies [min, mean, 50, 90, 95, 99, max] 442.281µs, 858.712µs, 854.673µs, 973.029µs, 1.017ms, 1.179ms, 10.485ms -Bytes In [total, mean] 1869632, 155.80 +Requests [total, rate, throughput] 12000, 100.01, 83.34 +Duration [total, attack, wait] 2m0s, 2m0s, 549.786µs +Latencies [min, mean, 50, 90, 95, 99, max] 389.965µs, 1.019ms, 1.081ms, 1.267ms, 1.325ms, 1.49ms, 5.198ms +Bytes In [total, mean] 1891642, 157.64 Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:12000 +Success [ratio] 83.33% +Status Codes [code:count] 0:2 200:10000 502:1998 Error Set: +Get "http://cafe.example.com/coffee": dial tcp 0.0.0.0:0->104.198.6.42:80: connect: connection refused +502 Bad Gateway ``` -![abrupt-scale-up-affinity-https-plus.png](abrupt-scale-up-affinity-https-plus.png) +![abrupt-scale-up-affinity-http-plus.png](abrupt-scale-up-affinity-http-plus.png) ### Scale Down Abruptly -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 804.798µs -Latencies [min, mean, 50, 90, 95, 99, max] 424.072µs, 837.824µs, 838.15µs, 963.636µs, 1.006ms, 1.123ms, 44.463ms -Bytes In [total, mean] 1940409, 161.70 +Duration [total, attack, wait] 2m0s, 2m0s, 1.184ms +Latencies [min, mean, 50, 90, 95, 99, max] 655.043µs, 1.2ms, 1.123ms, 1.298ms, 1.359ms, 1.514ms, 216.394ms +Bytes In [total, mean] 1838371, 153.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-down-affinity-http-plus.png](abrupt-scale-down-affinity-http-plus.png) +![abrupt-scale-down-affinity-https-plus.png](abrupt-scale-down-affinity-https-plus.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text -Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 685.714µs -Latencies [min, mean, 50, 90, 95, 99, max] 459.498µs, 865.342µs, 861.335µs, 990.99µs, 1.035ms, 1.151ms, 48.501ms -Bytes In [total, mean] 1869571, 155.80 +Requests [total, rate, throughput] 12000, 100.01, 83.34 +Duration [total, attack, wait] 2m0s, 2m0s, 1.092ms +Latencies [min, mean, 50, 90, 95, 99, max] 385.661µs, 999.806µs, 1.061ms, 1.236ms, 1.291ms, 1.419ms, 28.285ms +Bytes In [total, mean] 1892080, 157.67 Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:12000 +Success [ratio] 83.33% +Status Codes [code:count] 200:10000 502:2000 Error Set: +502 Bad Gateway ``` -![abrupt-scale-down-affinity-https-plus.png](abrupt-scale-down-affinity-https-plus.png) +![abrupt-scale-down-affinity-http-plus.png](abrupt-scale-down-affinity-http-plus.png) -## Multiple NGF Pods run per node Test Results +## Multiple NGINX Pods run per node Test Results ### Scale Up Gradually -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 499.02µs -Latencies [min, mean, 50, 90, 95, 99, max] 404.188µs, 862.699µs, 858.402µs, 1.003ms, 1.053ms, 1.359ms, 11.022ms -Bytes In [total, mean] 4862948, 162.10 +Duration [total, attack, wait] 5m0s, 5m0s, 1.143ms +Latencies [min, mean, 50, 90, 95, 99, max] 637.679µs, 1.198ms, 1.173ms, 1.371ms, 1.456ms, 1.718ms, 77.974ms +Bytes In [total, mean] 4625810, 154.19 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:30000 +Status Codes [code:count] 0:1 200:29999 Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->104.198.6.42:443: connect: connection refused ``` -![gradual-scale-up-http-plus.png](gradual-scale-up-http-plus.png) +![gradual-scale-up-https-plus.png](gradual-scale-up-https-plus.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 30000, 100.00, 100.00 -Duration [total, attack, wait] 5m0s, 5m0s, 917.782µs -Latencies [min, mean, 50, 90, 95, 99, max] 452.74µs, 884.958µs, 873.544µs, 1.017ms, 1.07ms, 1.42ms, 11.641ms -Bytes In [total, mean] 4682982, 156.10 +Duration [total, attack, wait] 5m0s, 5m0s, 1.107ms +Latencies [min, mean, 50, 90, 95, 99, max] 613.967µs, 1.173ms, 1.152ms, 1.363ms, 1.451ms, 1.738ms, 206.385ms +Bytes In [total, mean] 4800136, 160.00 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:30000 Error Set: ``` -![gradual-scale-up-https-plus.png](gradual-scale-up-https-plus.png) +![gradual-scale-up-http-plus.png](gradual-scale-up-http-plus.png) ### Scale Down Gradually #### Test: Send https /tea traffic ```text -Requests [total, rate, throughput] 96000, 100.00, 100.00 -Duration [total, attack, wait] 16m0s, 16m0s, 760.896µs -Latencies [min, mean, 50, 90, 95, 99, max] 433.285µs, 934.463µs, 905.034µs, 1.107ms, 1.202ms, 1.549ms, 83.045ms -Bytes In [total, mean] 14985575, 156.10 +Requests [total, rate, throughput] 96000, 100.00, 99.99 +Duration [total, attack, wait] 16m0s, 16m0s, 1.331ms +Latencies [min, mean, 50, 90, 95, 99, max] 103.225µs, 1.231ms, 1.207ms, 1.428ms, 1.508ms, 1.751ms, 219.8ms +Bytes In [total, mean] 14802221, 154.19 Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:96000 +Success [ratio] 99.99% +Status Codes [code:count] 0:6 200:95994 Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->104.198.6.42:443: connect: network is unreachable ``` ![gradual-scale-down-https-plus.png](gradual-scale-down-https-plus.png) @@ -205,77 +210,82 @@ Error Set: ```text Requests [total, rate, throughput] 96000, 100.00, 100.00 -Duration [total, attack, wait] 16m0s, 16m0s, 797.537µs -Latencies [min, mean, 50, 90, 95, 99, max] 389.802µs, 906.16µs, 872.26µs, 1.081ms, 1.271ms, 1.729ms, 78.489ms -Bytes In [total, mean] 15561579, 162.10 +Duration [total, attack, wait] 16m0s, 16m0s, 1.384ms +Latencies [min, mean, 50, 90, 95, 99, max] 53.523µs, 1.192ms, 1.165ms, 1.369ms, 1.441ms, 1.696ms, 259.683ms +Bytes In [total, mean] 15358929, 159.99 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% -Status Codes [code:count] 200:96000 +Status Codes [code:count] 0:4 200:95996 Error Set: +Get "http://cafe.example.com/coffee": dial tcp 0.0.0.0:0->104.198.6.42:80: connect: network is unreachable ``` ![gradual-scale-down-http-plus.png](gradual-scale-down-http-plus.png) ### Scale Up Abruptly -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text -Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 1.038ms -Latencies [min, mean, 50, 90, 95, 99, max] 438.072µs, 859.877µs, 851.049µs, 991.439µs, 1.042ms, 1.261ms, 9.194ms -Bytes In [total, mean] 1873263, 156.11 +Requests [total, rate, throughput] 12000, 100.01, 90.49 +Duration [total, attack, wait] 2m0s, 2m0s, 1.187ms +Latencies [min, mean, 50, 90, 95, 99, max] 406.379µs, 1.094ms, 1.113ms, 1.332ms, 1.412ms, 1.597ms, 5.752ms +Bytes In [total, mean] 1908208, 159.02 Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:12000 +Success [ratio] 90.48% +Status Codes [code:count] 0:2 200:10858 502:1140 Error Set: +Get "http://cafe.example.com/coffee": dial tcp 0.0.0.0:0->104.198.6.42:80: connect: connection refused +502 Bad Gateway ``` -![abrupt-scale-up-https-plus.png](abrupt-scale-up-https-plus.png) +![abrupt-scale-up-http-plus.png](abrupt-scale-up-http-plus.png) -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text -Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 745.836µs -Latencies [min, mean, 50, 90, 95, 99, max] 397.717µs, 825.498µs, 823.88µs, 955.33µs, 1.002ms, 1.198ms, 9.229ms -Bytes In [total, mean] 1945082, 162.09 +Requests [total, rate, throughput] 12000, 100.01, 98.82 +Duration [total, attack, wait] 2m0s, 2m0s, 1.183ms +Latencies [min, mean, 50, 90, 95, 99, max] 444.212µs, 1.165ms, 1.159ms, 1.365ms, 1.441ms, 1.656ms, 11.264ms +Bytes In [total, mean] 1848289, 154.02 Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:12000 +Success [ratio] 98.81% +Status Codes [code:count] 0:10 200:11857 502:133 Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->104.198.6.42:443: connect: connection refused +502 Bad Gateway ``` -![abrupt-scale-up-http-plus.png](abrupt-scale-up-http-plus.png) +![abrupt-scale-up-https-plus.png](abrupt-scale-up-https-plus.png) ### Scale Down Abruptly -#### Test: Send http /coffee traffic +#### Test: Send https /tea traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 853.74µs -Latencies [min, mean, 50, 90, 95, 99, max] 434.496µs, 857.503µs, 848.746µs, 975.447µs, 1.022ms, 1.187ms, 26.289ms -Bytes In [total, mean] 1945253, 162.10 +Duration [total, attack, wait] 2m0s, 2m0s, 997.033µs +Latencies [min, mean, 50, 90, 95, 99, max] 679.585µs, 1.238ms, 1.213ms, 1.418ms, 1.481ms, 1.653ms, 205.356ms +Bytes In [total, mean] 1850431, 154.20 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-down-http-plus.png](abrupt-scale-down-http-plus.png) +![abrupt-scale-down-https-plus.png](abrupt-scale-down-https-plus.png) -#### Test: Send https /tea traffic +#### Test: Send http /coffee traffic ```text Requests [total, rate, throughput] 12000, 100.01, 100.01 -Duration [total, attack, wait] 2m0s, 2m0s, 849.564µs -Latencies [min, mean, 50, 90, 95, 99, max] 453.708µs, 899.405µs, 881.221µs, 1.024ms, 1.074ms, 1.234ms, 8.51ms -Bytes In [total, mean] 1873266, 156.11 +Duration [total, attack, wait] 2m0s, 2m0s, 965.724µs +Latencies [min, mean, 50, 90, 95, 99, max] 616.281µs, 1.194ms, 1.185ms, 1.385ms, 1.451ms, 1.626ms, 45.327ms +Bytes In [total, mean] 1920085, 160.01 Bytes Out [total, mean] 0, 0.00 Success [ratio] 100.00% Status Codes [code:count] 200:12000 Error Set: ``` -![abrupt-scale-down-https-plus.png](abrupt-scale-down-https-plus.png) +![abrupt-scale-down-http-plus.png](abrupt-scale-down-http-plus.png) diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-oss.png index 450efa3f68..e7c740dfee 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-plus.png index f09d91c088..b1155c516f 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-oss.png index 450efa3f68..e7c740dfee 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-plus.png index f09d91c088..b1155c516f 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-oss.png index 1956882a7a..e486f93650 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-plus.png index 23d466b705..185145ebe4 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-oss.png index 1956882a7a..e486f93650 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-plus.png index 23d466b705..185145ebe4 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-down-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-oss.png index 4d7a5204e9..f3b77607c8 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-plus.png index 75b6e12b2c..9df426360b 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-oss.png index 4d7a5204e9..f3b77607c8 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-plus.png index 75b6e12b2c..9df426360b 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-oss.png index 08f89e08c5..bc1f6c52b4 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-plus.png index e0b62dc33f..814b43a769 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-oss.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-oss.png index 08f89e08c5..5772d2745f 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-oss.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-plus.png b/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-plus.png index e0b62dc33f..0a8af859bb 100644 Binary files a/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-plus.png and b/tests/results/zero-downtime-scale/edge/gradual-scale-up-https-plus.png differ