diff --git a/tests/results/dp-perf/2.0.0/2.0.0-oss.md b/tests/results/dp-perf/2.0.0/2.0.0-oss.md new file mode 100644 index 0000000000..452d9cb86f --- /dev/null +++ b/tests/results/dp-perf/2.0.0/2.0.0-oss.md @@ -0,0 +1,90 @@ +# Results + +## Test environment + +NGINX Plus: false + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Average latency slightly increased across all routing methods. + +## Test1: Running latte path based routing + +```text +Requests [total, rate, throughput] 30000, 1000.00, 999.97 +Duration [total, attack, wait] 30.001s, 30s, 855.194µs +Latencies [min, mean, 50, 90, 95, 99, max] 632.128µs, 875.548µs, 862.905µs, 983.833µs, 1.031ms, 1.148ms, 6.856ms +Bytes In [total, mean] 4800000, 160.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test2: Running coffee header based routing + +```text +Requests [total, rate, throughput] 30000, 999.98, 999.95 +Duration [total, attack, wait] 30.002s, 30.001s, 1.004ms +Latencies [min, mean, 50, 90, 95, 99, max] 692.343µs, 908.751µs, 893.265µs, 1.009ms, 1.054ms, 1.176ms, 30.581ms +Bytes In [total, mean] 4830000, 161.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test3: Running coffee query based routing + +```text +Requests [total, rate, throughput] 30000, 1000.02, 999.99 +Duration [total, attack, wait] 30s, 29.999s, 841.484µs +Latencies [min, mean, 50, 90, 95, 99, max] 695.394µs, 912.502µs, 898.684µs, 1.022ms, 1.068ms, 1.183ms, 10.847ms +Bytes In [total, mean] 5070000, 169.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test4: Running tea GET method based routing + +```text +Requests [total, rate, throughput] 30000, 1000.01, 999.98 +Duration [total, attack, wait] 30.001s, 30s, 955.507µs +Latencies [min, mean, 50, 90, 95, 99, max] 685.791µs, 902.297µs, 888.939µs, 1.01ms, 1.057ms, 1.166ms, 9.243ms +Bytes In [total, mean] 4740000, 158.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test5: Running tea POST method based routing + +```text +Requests [total, rate, throughput] 29999, 1000.00, 999.97 +Duration [total, attack, wait] 30s, 29.999s, 933.986µs +Latencies [min, mean, 50, 90, 95, 99, max] 666.013µs, 916.212µs, 887.892µs, 1.011ms, 1.06ms, 1.191ms, 207.34ms +Bytes In [total, mean] 4739842, 158.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:29999 +Error Set: +``` diff --git a/tests/results/dp-perf/2.0.0/2.0.0-plus.md b/tests/results/dp-perf/2.0.0/2.0.0-plus.md new file mode 100644 index 0000000000..cb0c67b4d1 --- /dev/null +++ b/tests/results/dp-perf/2.0.0/2.0.0-plus.md @@ -0,0 +1,90 @@ +# Results + +## Test environment + +NGINX Plus: true + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Average latency slightly increased across all routing methods. + +## Test1: Running latte path based routing + +```text +Requests [total, rate, throughput] 30000, 999.97, 999.82 +Duration [total, attack, wait] 30.005s, 30.001s, 4.397ms +Latencies [min, mean, 50, 90, 95, 99, max] 674.871µs, 1.095s, 991.559µs, 5.087s, 6.518s, 7.551s, 7.768s +Bytes In [total, mean] 4770000, 159.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test2: Running coffee header based routing + +```text +Requests [total, rate, throughput] 30000, 1000.04, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 1.129ms +Latencies [min, mean, 50, 90, 95, 99, max] 658.096µs, 960.704µs, 943.39µs, 1.091ms, 1.144ms, 1.34ms, 4.704ms +Bytes In [total, mean] 4800000, 160.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test3: Running coffee query based routing + +```text +Requests [total, rate, throughput] 30000, 1000.00, 999.97 +Duration [total, attack, wait] 30.001s, 30s, 903.354µs +Latencies [min, mean, 50, 90, 95, 99, max] 666.726µs, 962.973µs, 938.524µs, 1.094ms, 1.153ms, 1.375ms, 16.612ms +Bytes In [total, mean] 5040000, 168.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test4: Running tea GET method based routing + +```text +Requests [total, rate, throughput] 30000, 1000.04, 1000.01 +Duration [total, attack, wait] 30s, 29.999s, 905.444µs +Latencies [min, mean, 50, 90, 95, 99, max] 664.584µs, 941.233µs, 924.206µs, 1.079ms, 1.138ms, 1.324ms, 6.779ms +Bytes In [total, mean] 4710000, 157.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +## Test5: Running tea POST method based routing + +```text +Requests [total, rate, throughput] 30000, 1000.03, 999.99 +Duration [total, attack, wait] 30s, 29.999s, 1.099ms +Latencies [min, mean, 50, 90, 95, 99, max] 651.328µs, 944.335µs, 927.829µs, 1.081ms, 1.136ms, 1.29ms, 13.497ms +Bytes In [total, mean] 4710000, 157.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` diff --git a/tests/results/reconfig/2.0.0/2.0.0-oss.md b/tests/results/reconfig/2.0.0/2.0.0-oss.md new file mode 100644 index 0000000000..6e2443999b --- /dev/null +++ b/tests/results/reconfig/2.0.0/2.0.0-oss.md @@ -0,0 +1,150 @@ +# Results + +## Test environment + +NGINX Plus: false + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Expectations of the test suite have changed a bit due to the new architecture. +- Time to ready generally increases due to provisioning NGINX. +- Seeing some jumbled configuration issues, this has been brought to the attention of the NGINX Agent team. + +## Test 1: Resources exist before startup - NumResources 30 + +### Time to Ready + +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 15s + +### Event Batch Processing + +- Event Batch Total: 10 +- Event Batch Processing Average Time: 3ms +- Event Batch Processing distribution: + - 500.0ms: 10 + - 1000.0ms: 10 + - 5000.0ms: 10 + - 10000.0ms: 10 + - 30000.0ms: 10 + - +Infms: 10 + +### NGINX Error Logs + +## Test 1: Resources exist before startup - NumResources 150 + +### Time to Ready + +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 9s + +### Event Batch Processing + +- Event Batch Total: 10 +- Event Batch Processing Average Time: 6ms +- Event Batch Processing distribution: + - 500.0ms: 10 + - 1000.0ms: 10 + - 5000.0ms: 10 + - 10000.0ms: 10 + - 30000.0ms: 10 + - +Infms: 10 + +### NGINX Error Logs + +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 + +### Time to Ready + +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 21s + +### Event Batch Processing + +- Event Batch Total: 315 +- Event Batch Processing Average Time: 11ms +- Event Batch Processing distribution: + - 500.0ms: 315 + - 1000.0ms: 315 + - 5000.0ms: 315 + - 10000.0ms: 315 + - 30000.0ms: 315 + - +Infms: 315 + +### NGINX Error Logs +2025/06/02 16:12:16 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:666 +2025/06/02 16:12:16 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:1004 +2025/06/02 16:12:17 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:1113 +2025/06/02 16:12:20 [emerg] 8#8: pread() returned only 0 bytes instead of 4086 in /etc/nginx/conf.d/http.conf:715 + +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 + +### Time to Ready + +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 109s + +### Event Batch Processing + +- Event Batch Total: 1445 +- Event Batch Processing Average Time: 18ms +- Event Batch Processing distribution: + - 500.0ms: 1445 + - 1000.0ms: 1445 + - 5000.0ms: 1445 + - 10000.0ms: 1445 + - 30000.0ms: 1445 + - +Infms: 1445 + +### NGINX Error Logs +2025/06/02 16:16:12 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:197 +2025/06/02 16:16:13 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:1004 +2025/06/02 16:16:14 [emerg] 8#8: unknown directive "ts/ssl_keypair_reconfig_cafe-secret.pem" in /etc/nginx/conf.d/http.conf:189 +2025/06/02 16:16:15 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:1974 +2025/06/02 16:16:17 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:2639 +2025/06/02 16:16:18 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:3282 +2025/06/02 16:16:18 [emerg] 8#8: pread() returned only 0 bytes instead of 4089 in /etc/nginx/conf.d/http.conf:2416 +2025/06/02 16:16:21 [emerg] 8#8: pread() returned only 0 bytes instead of 4069 in /etc/nginx/conf.d/http.conf:193 +2025/06/02 16:16:21 [emerg] 8#8: pread() returned only 0 bytes instead of 4086 in /etc/nginx/conf.d/http.conf:905 +2025/06/02 16:16:25 [emerg] 8#8: invalid number of arguments in "zone" directive in /etc/nginx/conf.d/http.conf:5862 +2025/06/02 16:16:26 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:6949 +2025/06/02 16:16:27 [emerg] 8#8: pread() returned only 0 bytes instead of 4093 in /etc/nginx/conf.d/http.conf:2421 +2025/06/02 16:16:27 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:7315 +2025/06/02 16:16:27 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:7494 +2025/06/02 16:16:28 [emerg] 8#8: pread() returned only 0 bytes instead of 4081 in /etc/nginx/conf.d/http.conf:3875 +2025/06/02 16:16:28 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:4364 +2025/06/02 16:16:28 [emerg] 8#8: pread() returned only 0 bytes instead of 4093 in /etc/nginx/conf.d/http.conf:4847 +2025/06/02 16:16:29 [emerg] 8#8: pread() returned only 0 bytes instead of 4094 in /etc/nginx/conf.d/http.conf:632 +2025/06/02 16:16:29 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:8514 +2025/06/02 16:16:29 [emerg] 8#8: pread() returned only 0 bytes instead of 4085 in /etc/nginx/conf.d/http.conf:7409 +2025/06/02 16:16:29 [emerg] 8#8: pread() returned only 0 bytes instead of 4096 in /etc/nginx/conf.d/http.conf:6299 +2025/06/02 16:16:31 [emerg] 8#8: pread() returned only 0 bytes instead of 4084 in /etc/nginx/conf.d/http.conf:6343 +2025/06/02 16:16:33 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:10970 +2025/06/02 16:16:35 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:4495 +2025/06/02 16:16:35 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:11951 +2025/06/02 16:16:36 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:12438 +2025/06/02 16:16:37 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:8234 +2025/06/02 16:16:39 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:13878 +2025/06/02 16:16:39 [emerg] 8#8: pread() returned only 0 bytes instead of 4076 in /etc/nginx/conf.d/http.conf:2329 +2025/06/02 16:16:41 [emerg] 8#8: unexpected end of file, expecting "}" in /etc/nginx/conf.d/http.conf:15054 +2025/06/02 16:16:41 [emerg] 8#8: pread() returned only 0 bytes instead of 4095 in /etc/nginx/conf.d/http.conf:9862 +2025/06/02 16:16:42 [emerg] 8#8: pread() returned only 0 bytes instead of 4079 in /etc/nginx/conf.d/http.conf:14393 +2025/06/02 16:16:42 [emerg] 8#8: pread() returned only 0 bytes instead of 2780 in /etc/nginx/conf.d/http.conf:15535 +2025/06/02 16:16:43 [emerg] 8#8: pread() returned only 0 bytes instead of 4083 in /etc/nginx/conf.d/http.conf:5118 +2025/06/02 16:16:43 [emerg] 8#8: unexpected end of file, expecting ";" or "}" in /etc/nginx/conf.d/http.conf:16167 diff --git a/tests/results/reconfig/2.0.0/2.0.0-plus.md b/tests/results/reconfig/2.0.0/2.0.0-plus.md new file mode 100644 index 0000000000..c3df77ea82 --- /dev/null +++ b/tests/results/reconfig/2.0.0/2.0.0-plus.md @@ -0,0 +1,110 @@ +# Results + +## Test environment + +NGINX Plus: true + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Expectations of the test suite have changed a bit due to the new architecture. +- Time to ready generally increases due to provisioning NGINX. + +## Test 1: Resources exist before startup - NumResources 30 + +### Time to Ready + +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 18s + +### Event Batch Processing + +- Event Batch Total: 9 +- Event Batch Processing Average Time: 17ms +- Event Batch Processing distribution: + - 500.0ms: 9 + - 1000.0ms: 9 + - 5000.0ms: 9 + - 10000.0ms: 9 + - 30000.0ms: 9 + - +Infms: 9 + +### NGINX Error Logs + +## Test 1: Resources exist before startup - NumResources 150 + +### Time to Ready + +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 10s + +### Event Batch Processing + +- Event Batch Total: 8 +- Event Batch Processing Average Time: 35ms +- Event Batch Processing distribution: + - 500.0ms: 8 + - 1000.0ms: 8 + - 5000.0ms: 8 + - 10000.0ms: 8 + - 30000.0ms: 8 + - +Infms: 8 + +### NGINX Error Logs + +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 + +### Time to Ready + +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 20s + +### Event Batch Processing + +- Event Batch Total: 213 +- Event Batch Processing Average Time: 32ms +- Event Batch Processing distribution: + - 500.0ms: 204 + - 1000.0ms: 213 + - 5000.0ms: 213 + - 10000.0ms: 213 + - 30000.0ms: 213 + - +Infms: 213 + +### NGINX Error Logs + +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 + +### Time to Ready + +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 100s + +### Event Batch Processing + +- Event Batch Total: 1013 +- Event Batch Processing Average Time: 33ms +- Event Batch Processing distribution: + - 500.0ms: 986 + - 1000.0ms: 1000 + - 5000.0ms: 1013 + - 10000.0ms: 1013 + - 30000.0ms: 1013 + - +Infms: 1013 + +### NGINX Error Logs diff --git a/tests/results/scale/2.0.0/2.0.0-oss.md b/tests/results/scale/2.0.0/2.0.0-oss.md new file mode 100644 index 0000000000..28d436fb7e --- /dev/null +++ b/tests/results/scale/2.0.0/2.0.0-oss.md @@ -0,0 +1,156 @@ +# Results + +## Test environment + +NGINX Plus: false + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Expectations of the test suite have changed a bit due to the new architecture. +- Multiple errors in the contol plane relating to updating Gateway status after it has been modified, also seen in previous releases. +- Memory usage has increased (likely due to Agent). +- TTR also increased. +- Latency increased. + +## Test TestScale_Listeners + +### Event Batch Processing + +- Total: 325 +- Average Time: 6ms +- Event Batch Processing distribution: + - 500.0ms: 325 + - 1000.0ms: 325 + - 5000.0ms: 325 + - 10000.0ms: 325 + - 30000.0ms: 325 + - +Infms: 325 + +### Errors + +- NGF errors: 15 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_Listeners) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPSListeners + +### Event Batch Processing + +- Total: 389 +- Average Time: 19ms +- Event Batch Processing distribution: + - 500.0ms: 389 + - 1000.0ms: 389 + - 5000.0ms: 389 + - 10000.0ms: 389 + - 30000.0ms: 389 + - +Infms: 389 + +### Errors + +- NGF errors: 18 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_HTTPSListeners) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPRoutes + +### Event Batch Processing + +- Total: 1009 +- Average Time: 93ms +- Event Batch Processing distribution: + - 500.0ms: 1009 + - 1000.0ms: 1009 + - 5000.0ms: 1009 + - 10000.0ms: 1009 + - 30000.0ms: 1009 + - +Infms: 1009 + +### Errors + +- NGF errors: 0 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_HTTPRoutes) for more details. +The logs are attached only if there are errors. + +## Test TestScale_UpstreamServers + +### Event Batch Processing + +- Total: 96 +- Average Time: 82ms +- Event Batch Processing distribution: + - 500.0ms: 96 + - 1000.0ms: 96 + - 5000.0ms: 96 + - 10000.0ms: 96 + - 30000.0ms: 96 + - +Infms: 96 + +### Errors + +- NGF errors: 2 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_UpstreamServers) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPMatches + +```text +Requests [total, rate, throughput] 30000, 1000.01, 999.98 +Duration [total, attack, wait] 30s, 30s, 870.558µs +Latencies [min, mean, 50, 90, 95, 99, max] 661.923µs, 900.15µs, 888.179µs, 1.003ms, 1.048ms, 1.167ms, 8.488ms +Bytes In [total, mean] 4800000, 160.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` +```text +Requests [total, rate, throughput] 30000, 1000.03, 1000.00 +Duration [total, attack, wait] 30s, 29.999s, 918.805µs +Latencies [min, mean, 50, 90, 95, 99, max] 792.213µs, 1.031ms, 1.012ms, 1.128ms, 1.173ms, 1.338ms, 12.464ms +Bytes In [total, mean] 4800000, 160.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` diff --git a/tests/results/scale/2.0.0/2.0.0-plus.md b/tests/results/scale/2.0.0/2.0.0-plus.md new file mode 100644 index 0000000000..d1dd707be4 --- /dev/null +++ b/tests/results/scale/2.0.0/2.0.0-plus.md @@ -0,0 +1,156 @@ +# Results + +## Test environment + +NGINX Plus: true + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Expectations of the test suite have changed a bit due to the new architecture. +- Multiple errors in the contol plane relating to updating Gateway status after it has been modified, also seen in previous releases. +- Even more "no live upstream" logs than previously. +- Memory usage has increased (likely due to Agent). +- Latency decreased. + +## Test TestScale_Listeners + +### Event Batch Processing + +- Total: 323 +- Average Time: 122ms +- Event Batch Processing distribution: + - 500.0ms: 279 + - 1000.0ms: 323 + - 5000.0ms: 323 + - 10000.0ms: 323 + - 30000.0ms: 323 + - +Infms: 323 + +### Errors + +- NGF errors: 14 +- NGF container restarts: 0 +- NGINX errors: 170 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_Listeners) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPSListeners + +### Event Batch Processing + +- Total: 389 +- Average Time: 115ms +- Event Batch Processing distribution: + - 500.0ms: 346 + - 1000.0ms: 389 + - 5000.0ms: 389 + - 10000.0ms: 389 + - 30000.0ms: 389 + - +Infms: 389 + +### Errors + +- NGF errors: 8 +- NGF container restarts: 0 +- NGINX errors: 163 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_HTTPSListeners) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPRoutes + +### Event Batch Processing + +- Total: 1009 +- Average Time: 161ms +- Event Batch Processing distribution: + - 500.0ms: 1009 + - 1000.0ms: 1009 + - 5000.0ms: 1009 + - 10000.0ms: 1009 + - 30000.0ms: 1009 + - +Infms: 1009 + +### Errors + +- NGF errors: 0 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_HTTPRoutes) for more details. +The logs are attached only if there are errors. + +## Test TestScale_UpstreamServers + +### Event Batch Processing + +- Total: 46 +- Average Time: 376ms +- Event Batch Processing distribution: + - 500.0ms: 32 + - 1000.0ms: 45 + - 5000.0ms: 46 + - 10000.0ms: 46 + - 30000.0ms: 46 + - +Infms: 46 + +### Errors + +- NGF errors: 1 +- NGF container restarts: 0 +- NGINX errors: 0 +- NGINX container restarts: 0 + +### Graphs and Logs + +See [output directory](./TestScale_UpstreamServers) for more details. +The logs are attached only if there are errors. + +## Test TestScale_HTTPMatches + +```text +Requests [total, rate, throughput] 30000, 1000.04, 1000.01 +Duration [total, attack, wait] 30s, 29.999s, 662.014µs +Latencies [min, mean, 50, 90, 95, 99, max] 440.386µs, 660.712µs, 636.127µs, 764.866µs, 820.872µs, 988.222µs, 14.635ms +Bytes In [total, mean] 4770000, 159.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` +```text +Requests [total, rate, throughput] 30000, 1000.03, 1000.01 +Duration [total, attack, wait] 30s, 29.999s, 780.099µs +Latencies [min, mean, 50, 90, 95, 99, max] 576.932µs, 804.634µs, 772.311µs, 937.316µs, 1.038ms, 1.207ms, 12.889ms +Bytes In [total, mean] 4770000, 159.00 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-oss.png new file mode 100644 index 0000000000..3da988369e Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-plus.png new file mode 100644 index 0000000000..be9f6a2a4b Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/cpu-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-oss.png new file mode 100644 index 0000000000..efe1c0e618 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-plus.png new file mode 100644 index 0000000000..f6371c5859 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/memory-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-oss.png new file mode 100644 index 0000000000..f10db473af Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-plus.png new file mode 100644 index 0000000000..28a33074ea Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPRoutes/ttr-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-oss.png new file mode 100644 index 0000000000..c3f4d7fe08 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-plus.png new file mode 100644 index 0000000000..8ce868d168 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/cpu-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-oss.png new file mode 100644 index 0000000000..637b7262e9 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-plus.png new file mode 100644 index 0000000000..58acb0a73f Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/memory-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-oss.log b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-oss.log new file mode 100644 index 0000000000..cefbaa19f4 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-oss.log @@ -0,0 +1,18 @@ +{"level":"debug","ts":"2025-06-02T15:38:35Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:12Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:22Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:23Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:24Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:27Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:27Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:29Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:30Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:31Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:32Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:32Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:34Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:35Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:40Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:41Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:49Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:51Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-plus.log b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-plus.log new file mode 100644 index 0000000000..5989dff470 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ngf-plus.log @@ -0,0 +1,8 @@ +{"level":"debug","ts":"2025-06-02T15:37:51Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gatewayclasses.gateway.networking.k8s.io \"nginx\": the object has been modified; please apply your changes to the latest version and try again","namespace":"","name":"nginx","kind":"GatewayClass"} +{"level":"debug","ts":"2025-06-02T15:38:24Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:16Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:19Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:26Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:46Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:39:48Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:40:18Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/nginx-plus.log b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/nginx-plus.log new file mode 100644 index 0000000000..91237eba84 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/nginx-plus.log @@ -0,0 +1,163 @@ +2025/06/02 15:39:06 [error] 150#150: *43 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:39:06 [error] 151#151: *44 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:39:06 [error] 153#153: *45 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:39:06 [error] 154#154: *46 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:39:07 [error] 186#186: *60 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:39:07 [error] 187#187: *61 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:39:07 [error] 189#189: *62 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:39:07 [error] 190#190: *63 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:39:08 [error] 222#222: *78 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:39:08 [error] 223#223: *79 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:39:08 [error] 225#225: *80 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:39:08 [error] 226#226: *81 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:39:08 [error] 258#258: *97 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:39:09 [error] 259#259: *98 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:39:09 [error] 261#261: *99 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:39:09 [error] 262#262: *100 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:39:09 [error] 294#294: *117 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:39:09 [error] 295#295: *118 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:39:10 [error] 297#297: *119 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:39:10 [error] 298#298: *120 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:39:10 [error] 330#330: *138 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:39:10 [error] 331#331: *139 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:39:10 [error] 333#333: *141 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:39:10 [error] 334#334: *142 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:39:11 [error] 335#335: *143 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:39:12 [error] 368#368: *162 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:39:12 [error] 369#369: *163 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:39:12 [error] 371#371: *164 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:39:12 [error] 372#372: *165 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:39:13 [error] 373#373: *166 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:39:13 [error] 404#404: *187 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:39:13 [error] 405#405: *188 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:39:13 [error] 407#407: *189 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:39:13 [error] 408#408: *190 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:39:14 [error] 409#409: *191 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:39:14 [error] 440#440: *212 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:39:14 [error] 441#441: *213 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:39:14 [error] 443#443: *214 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:39:14 [error] 444#444: *215 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:39:14 [error] 445#445: *216 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:39:15 [error] 476#476: *229 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:39:15 [error] 477#477: *230 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:39:15 [error] 479#479: *231 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:39:15 [error] 480#480: *233 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:39:16 [error] 514#514: *256 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:39:16 [error] 515#515: *257 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:39:16 [error] 517#517: *258 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:39:16 [error] 518#518: *259 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:39:17 [error] 550#550: *286 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:39:17 [error] 551#551: *287 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:39:17 [error] 553#553: *288 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:39:17 [error] 554#554: *289 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:39:18 [error] 586#586: *314 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/06/02 15:39:18 [error] 587#587: *315 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/06/02 15:39:18 [error] 589#589: *316 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/06/02 15:39:18 [error] 590#590: *317 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 13.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-13_80/", host: "13.example.com" +2025/06/02 15:39:19 [error] 622#622: *344 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:39:19 [error] 623#623: *345 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:39:19 [error] 625#625: *346 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:39:19 [error] 626#626: *347 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:39:20 [error] 658#658: *374 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:39:20 [error] 659#659: *375 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:39:20 [error] 661#661: *376 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:39:20 [error] 662#662: *377 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:39:20 [error] 663#663: *378 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:39:21 [error] 711#711: *407 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:39:21 [error] 696#696: *408 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:39:21 [error] 697#697: *409 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:39:21 [error] 699#699: *410 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:39:21 [error] 700#700: *411 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:39:22 [error] 732#732: *441 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:39:22 [error] 733#733: *442 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:39:22 [error] 735#735: *443 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:39:22 [error] 736#736: *444 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:39:22 [error] 737#737: *445 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:39:23 [error] 771#771: *476 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:39:23 [error] 768#768: *477 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:39:23 [error] 769#769: *478 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:39:23 [error] 772#772: *479 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:39:24 [error] 773#773: *480 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:39:24 [error] 804#804: *512 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:39:24 [error] 805#805: *513 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:39:24 [error] 807#807: *514 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:39:25 [error] 808#808: *515 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:39:25 [error] 840#840: *550 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/06/02 15:39:26 [error] 841#841: *551 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/06/02 15:39:26 [error] 843#843: *552 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/06/02 15:39:26 [error] 844#844: *553 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 20.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-20_80/", host: "20.example.com" +2025/06/02 15:39:26 [error] 878#878: *587 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 21.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-21_80/", host: "21.example.com" +2025/06/02 15:39:27 [error] 879#879: *588 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 21.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-21_80/", host: "21.example.com" +2025/06/02 15:39:27 [error] 881#881: *589 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 21.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-21_80/", host: "21.example.com" +2025/06/02 15:39:27 [error] 882#882: *590 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 21.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-21_80/", host: "21.example.com" +2025/06/02 15:39:28 [error] 950#950: *659 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 23.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-23_80/", host: "23.example.com" +2025/06/02 15:39:28 [error] 951#951: *660 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 23.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-23_80/", host: "23.example.com" +2025/06/02 15:39:28 [error] 953#953: *661 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 23.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-23_80/", host: "23.example.com" +2025/06/02 15:39:28 [error] 954#954: *662 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 23.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-23_80/", host: "23.example.com" +2025/06/02 15:39:33 [error] 1132#1132: *849 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 28.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-28_80/", host: "28.example.com" +2025/06/02 15:39:36 [error] 1278#1278: *1017 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 32.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-32_80/", host: "32.example.com" +2025/06/02 15:39:37 [error] 1314#1314: *1059 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 33.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/06/02 15:39:39 [error] 1350#1350: *1107 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 34.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-34_80/", host: "34.example.com" +2025/06/02 15:39:40 [error] 1386#1386: *1152 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 35.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/06/02 15:39:41 [error] 1424#1424: *1202 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 36.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/06/02 15:39:42 [error] 1460#1460: *1239 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 37.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/06/02 15:39:42 [error] 1461#1461: *1251 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 37.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/06/02 15:39:43 [error] 1497#1497: *1315 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 38.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/06/02 15:39:44 [error] 1533#1533: *1362 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 39.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/06/02 15:39:46 [error] 1571#1571: *1412 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 40.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/06/02 15:39:47 [error] 1607#1607: *1466 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 41.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/06/02 15:39:48 [error] 1643#1643: *1514 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 42.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/06/02 15:39:49 [error] 1644#1644: *1528 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 42.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/06/02 15:39:50 [error] 1679#1679: *1573 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 43.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/06/02 15:39:50 [error] 1680#1680: *1588 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 43.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/06/02 15:39:51 [error] 1717#1717: *1629 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 44.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/06/02 15:39:51 [error] 1718#1718: *1645 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 44.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/06/02 15:39:53 [error] 1753#1753: *1685 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 45.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/06/02 15:39:53 [error] 1754#1754: *1699 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 45.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/06/02 15:39:54 [error] 1790#1790: *1744 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 46.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/06/02 15:39:54 [error] 1791#1791: *1759 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 46.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/06/02 15:39:55 [error] 1826#1826: *1806 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/06/02 15:39:55 [error] 1827#1827: *1820 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 47.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/06/02 15:39:56 [error] 1864#1864: *1864 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 48.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/06/02 15:39:57 [error] 1865#1865: *1878 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 48.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/06/02 15:39:58 [error] 1900#1900: *1926 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/06/02 15:39:58 [error] 1901#1901: *1939 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 49.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/06/02 15:40:00 [error] 1936#1936: *1982 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:40:00 [error] 1937#1937: *1993 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:40:00 [error] 1939#1939: *2003 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:40:01 [error] 1974#1974: *2052 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 51.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/06/02 15:40:01 [error] 1975#1975: *2065 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 51.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/06/02 15:40:03 [error] 2013#2013: *2119 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 52.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/06/02 15:40:03 [error] 2010#2010: *2133 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 52.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/06/02 15:40:05 [error] 2046#2046: *2182 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 53.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/06/02 15:40:05 [error] 2047#2047: *2195 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 53.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/06/02 15:40:06 [error] 2084#2084: *2249 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:40:07 [error] 2085#2085: *2263 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:40:07 [error] 2087#2087: *2276 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:40:08 [error] 2120#2120: *2315 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:40:09 [error] 2121#2121: *2330 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:40:09 [error] 2123#2123: *2342 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:40:10 [error] 2156#2156: *2381 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:40:10 [error] 2157#2157: *2394 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:40:10 [error] 2159#2159: *2408 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:40:11 [error] 2194#2194: *2453 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:40:11 [error] 2195#2195: *2466 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:40:12 [error] 2197#2197: *2480 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:40:13 [error] 2230#2230: *2526 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:40:13 [error] 2231#2231: *2541 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:40:13 [error] 2233#2233: *2556 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:40:14 [error] 2266#2266: *2599 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:40:14 [error] 2267#2267: *2613 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:40:14 [error] 2269#2269: *2628 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:40:15 [error] 2302#2302: *2670 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:40:15 [error] 2303#2303: *2685 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:40:15 [error] 2305#2305: *2701 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:40:17 [error] 2340#2340: *2757 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:40:17 [error] 2341#2341: *2772 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:40:17 [error] 2343#2343: *2787 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:40:19 [error] 2376#2376: *2828 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:40:19 [error] 2377#2377: *2842 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:40:19 [error] 2379#2379: *2856 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:40:21 [error] 2412#2412: *2908 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/06/02 15:40:21 [error] 2413#2413: *2924 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/06/02 15:40:21 [error] 2415#2415: *2940 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/2.0", upstream: "http://scale_backend-63_80/", host: "63.example.com" diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-oss.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-oss.png new file mode 100644 index 0000000000..e0d765de50 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-plus.png b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-plus.png new file mode 100644 index 0000000000..daf4520bbe Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_HTTPSListeners/ttr-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/cpu-oss.png b/tests/results/scale/2.0.0/TestScale_Listeners/cpu-oss.png new file mode 100644 index 0000000000..3169e8ad97 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/cpu-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/cpu-plus.png b/tests/results/scale/2.0.0/TestScale_Listeners/cpu-plus.png new file mode 100644 index 0000000000..a8f4eafb45 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/cpu-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/memory-oss.png b/tests/results/scale/2.0.0/TestScale_Listeners/memory-oss.png new file mode 100644 index 0000000000..e6149e51cd Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/memory-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/memory-plus.png b/tests/results/scale/2.0.0/TestScale_Listeners/memory-plus.png new file mode 100644 index 0000000000..5b26fb2906 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/memory-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/ngf-oss.log b/tests/results/scale/2.0.0/TestScale_Listeners/ngf-oss.log new file mode 100644 index 0000000000..69105aa971 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_Listeners/ngf-oss.log @@ -0,0 +1,15 @@ +{"level":"debug","ts":"2025-06-02T15:34:33Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:36Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:37Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:45Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:47Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:49Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:51Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:52Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:53Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:54Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"error","ts":"2025-06-02T15:35:54Z","logger":"statusUpdater","msg":"Failed to update status","namespace":"scale","name":"gateway","kind":"","error":"timed out waiting for the condition","stacktrace":"github.com/nginx/nginx-gateway-fabric/internal/controller/status.(*Updater).writeStatuses\n\t/home/runner/work/nginx-gateway-fabric/nginx-gateway-fabric/internal/controller/status/updater.go:112\ngithub.com/nginx/nginx-gateway-fabric/internal/controller/status.(*Updater).Update\n\t/home/runner/work/nginx-gateway-fabric/nginx-gateway-fabric/internal/controller/status/updater.go:83\ngithub.com/nginx/nginx-gateway-fabric/internal/controller/status.(*LeaderAwareGroupUpdater).UpdateGroup\n\t/home/runner/work/nginx-gateway-fabric/nginx-gateway-fabric/internal/controller/status/leader_aware_group_updater.go:54\ngithub.com/nginx/nginx-gateway-fabric/internal/controller.(*eventHandlerImpl).updateStatuses\n\t/home/runner/work/nginx-gateway-fabric/nginx-gateway-fabric/internal/controller/handler.go:373\ngithub.com/nginx/nginx-gateway-fabric/internal/controller.(*eventHandlerImpl).waitForStatusUpdates\n\t/home/runner/work/nginx-gateway-fabric/nginx-gateway-fabric/internal/controller/handler.go:273"} +{"level":"debug","ts":"2025-06-02T15:35:55Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:56Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:57Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:58Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/ngf-plus.log b/tests/results/scale/2.0.0/TestScale_Listeners/ngf-plus.log new file mode 100644 index 0000000000..eef85b5d2e --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_Listeners/ngf-plus.log @@ -0,0 +1,14 @@ +{"level":"debug","ts":"2025-06-02T15:33:59Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:04Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:08Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:12Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:12Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:13Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:13Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:13Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:14Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:18Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:19Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:20Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:21Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} +{"level":"debug","ts":"2025-06-02T15:35:41Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/nginx-plus.log b/tests/results/scale/2.0.0/TestScale_Listeners/nginx-plus.log new file mode 100644 index 0000000000..3f38bcc797 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_Listeners/nginx-plus.log @@ -0,0 +1,170 @@ +2025/06/02 15:34:55 [error] 136#136: *44 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:34:55 [error] 137#137: *45 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:34:55 [error] 139#139: *46 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:34:55 [error] 140#140: *47 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 1.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-1_80/", host: "1.example.com" +2025/06/02 15:34:55 [error] 156#156: *60 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:34:56 [error] 157#157: *61 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:34:56 [error] 159#159: *62 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:34:56 [error] 160#160: *63 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:34:56 [error] 161#161: *64 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 2.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-2_80/", host: "2.example.com" +2025/06/02 15:34:56 [error] 174#174: *77 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:34:56 [error] 175#175: *78 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:34:57 [error] 177#177: *79 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:34:57 [error] 178#178: *80 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 3.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-3_80/", host: "3.example.com" +2025/06/02 15:34:57 [error] 193#193: *94 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:34:57 [error] 192#192: *95 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:34:57 [error] 195#195: *96 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:34:57 [error] 196#196: *97 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 4.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-4_80/", host: "4.example.com" +2025/06/02 15:34:58 [error] 210#210: *112 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:34:58 [error] 211#211: *113 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:34:58 [error] 213#213: *114 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:34:58 [error] 214#214: *115 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:34:58 [error] 215#215: *116 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 5.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-5_80/", host: "5.example.com" +2025/06/02 15:34:59 [error] 228#228: *132 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:34:59 [error] 229#229: *133 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:34:59 [error] 231#231: *134 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:34:59 [error] 232#232: *135 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:34:59 [error] 233#233: *136 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 6.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-6_80/", host: "6.example.com" +2025/06/02 15:35:00 [error] 246#246: *150 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 7.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-7_80/", host: "7.example.com" +2025/06/02 15:35:00 [error] 266#266: *171 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:35:01 [error] 267#267: *172 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:35:01 [error] 269#269: *173 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:35:01 [error] 270#270: *174 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:35:01 [error] 271#271: *175 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 8.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-8_80/", host: "8.example.com" +2025/06/02 15:35:01 [error] 284#284: *194 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:35:01 [error] 285#285: *195 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:35:02 [error] 287#287: *196 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:35:02 [error] 288#288: *197 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:35:02 [error] 289#289: *198 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 9.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-9_80/", host: "9.example.com" +2025/06/02 15:35:03 [error] 302#302: *210 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:35:03 [error] 303#303: *211 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:35:03 [error] 305#305: *212 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:35:03 [error] 306#306: *213 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 10.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-10_80/", host: "10.example.com" +2025/06/02 15:35:04 [error] 320#320: *234 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:35:04 [error] 321#321: *235 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:35:04 [error] 323#323: *236 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:35:04 [error] 324#324: *237 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 11.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-11_80/", host: "11.example.com" +2025/06/02 15:35:04 [error] 338#338: *259 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:35:05 [error] 339#339: *260 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:35:05 [error] 341#341: *261 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:35:05 [error] 342#342: *262 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 12.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-12_80/", host: "12.example.com" +2025/06/02 15:35:06 [error] 376#376: *309 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:35:06 [error] 377#377: *310 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:35:06 [error] 379#379: *311 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:35:06 [error] 380#380: *312 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 14.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-14_80/", host: "14.example.com" +2025/06/02 15:35:07 [error] 394#394: *337 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:35:07 [error] 395#395: *338 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:35:07 [error] 397#397: *339 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:35:07 [error] 398#398: *340 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:35:07 [error] 399#399: *341 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 15.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-15_80/", host: "15.example.com" +2025/06/02 15:35:08 [error] 412#412: *367 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:35:08 [error] 413#413: *368 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:35:08 [error] 415#415: *369 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:35:08 [error] 416#416: *370 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 16.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-16_80/", host: "16.example.com" +2025/06/02 15:35:09 [error] 430#430: *398 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:35:09 [error] 431#431: *399 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:35:09 [error] 433#433: *400 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:35:09 [error] 434#434: *401 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:35:09 [error] 435#435: *402 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 17.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-17_80/", host: "17.example.com" +2025/06/02 15:35:10 [error] 448#448: *430 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:35:10 [error] 449#449: *431 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:35:10 [error] 451#451: *432 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:35:10 [error] 452#452: *433 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:35:10 [error] 453#453: *435 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 18.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-18_80/", host: "18.example.com" +2025/06/02 15:35:11 [error] 468#468: *464 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:35:11 [error] 469#469: *465 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:35:11 [error] 471#471: *466 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:35:11 [error] 472#472: *467 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:35:11 [error] 473#473: *468 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 19.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-19_80/", host: "19.example.com" +2025/06/02 15:35:14 [error] 558#558: *620 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 24.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-24_80/", host: "24.example.com" +2025/06/02 15:35:15 [error] 576#576: *653 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 25.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-25_80/", host: "25.example.com" +2025/06/02 15:35:15 [error] 596#596: *687 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 26.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-26_80/", host: "26.example.com" +2025/06/02 15:35:16 [error] 614#614: *723 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 27.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-27_80/", host: "27.example.com" +2025/06/02 15:35:18 [error] 652#652: *813 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 29.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-29_80/", host: "29.example.com" +2025/06/02 15:35:19 [error] 670#670: *850 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 30.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-30_80/", host: "30.example.com" +2025/06/02 15:35:20 [error] 688#688: *886 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 31.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-31_80/", host: "31.example.com" +2025/06/02 15:35:20 [error] 708#708: *933 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 32.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-32_80/", host: "32.example.com" +2025/06/02 15:35:22 [error] 726#726: *968 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 33.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/06/02 15:35:22 [error] 727#727: *981 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 33.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-33_80/", host: "33.example.com" +2025/06/02 15:35:23 [error] 744#744: *1010 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 34.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-34_80/", host: "34.example.com" +2025/06/02 15:35:23 [error] 745#745: *1023 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 34.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-34_80/", host: "34.example.com" +2025/06/02 15:35:24 [error] 762#762: *1057 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 35.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/06/02 15:35:24 [error] 763#763: *1069 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 35.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-35_80/", host: "35.example.com" +2025/06/02 15:35:25 [error] 780#780: *1103 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 36.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/06/02 15:35:25 [error] 781#781: *1117 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 36.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-36_80/", host: "36.example.com" +2025/06/02 15:35:26 [error] 800#800: *1146 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 37.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/06/02 15:35:26 [error] 801#801: *1159 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 37.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-37_80/", host: "37.example.com" +2025/06/02 15:35:27 [error] 818#818: *1197 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 38.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/06/02 15:35:27 [error] 819#819: *1211 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 38.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-38_80/", host: "38.example.com" +2025/06/02 15:35:28 [error] 836#836: *1242 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 39.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/06/02 15:35:28 [error] 837#837: *1253 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 39.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-39_80/", host: "39.example.com" +2025/06/02 15:35:28 [error] 854#854: *1296 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 40.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/06/02 15:35:29 [error] 855#855: *1309 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 40.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-40_80/", host: "40.example.com" +2025/06/02 15:35:30 [error] 872#872: *1345 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 41.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/06/02 15:35:30 [error] 873#873: *1358 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 41.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-41_80/", host: "41.example.com" +2025/06/02 15:35:31 [error] 892#892: *1396 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 42.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/06/02 15:35:31 [error] 893#893: *1408 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 42.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-42_80/", host: "42.example.com" +2025/06/02 15:35:33 [error] 910#910: *1448 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 43.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/06/02 15:35:33 [error] 911#911: *1460 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 43.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-43_80/", host: "43.example.com" +2025/06/02 15:35:35 [error] 928#928: *1500 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 44.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/06/02 15:35:35 [error] 929#929: *1512 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 44.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-44_80/", host: "44.example.com" +2025/06/02 15:35:36 [error] 948#948: *1556 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/06/02 15:35:36 [error] 949#949: *1566 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/06/02 15:35:36 [error] 951#951: *1578 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 45.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-45_80/", host: "45.example.com" +2025/06/02 15:35:37 [error] 966#966: *1612 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 46.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/06/02 15:35:37 [error] 967#967: *1627 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 46.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-46_80/", host: "46.example.com" +2025/06/02 15:35:38 [error] 984#984: *1670 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/06/02 15:35:38 [error] 985#985: *1681 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/06/02 15:35:39 [error] 987#987: *1695 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 47.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-47_80/", host: "47.example.com" +2025/06/02 15:35:40 [error] 1002#1002: *1727 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/06/02 15:35:40 [error] 1003#1003: *1742 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 48.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-48_80/", host: "48.example.com" +2025/06/02 15:35:41 [error] 1022#1022: *1786 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/06/02 15:35:41 [error] 1023#1023: *1799 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/06/02 15:35:41 [error] 1025#1025: *1812 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 49.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-49_80/", host: "49.example.com" +2025/06/02 15:35:43 [error] 1040#1040: *1846 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:35:43 [error] 1041#1041: *1860 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:35:43 [error] 1043#1043: *1875 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 50.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-50_80/", host: "50.example.com" +2025/06/02 15:35:44 [error] 1058#1058: *1907 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/06/02 15:35:44 [error] 1059#1059: *1920 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/06/02 15:35:44 [error] 1061#1061: *1935 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 51.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-51_80/", host: "51.example.com" +2025/06/02 15:35:46 [error] 1078#1078: *1977 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 52.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/06/02 15:35:46 [error] 1079#1079: *1991 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 52.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-52_80/", host: "52.example.com" +2025/06/02 15:35:48 [error] 1096#1096: *2029 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/06/02 15:35:48 [error] 1097#1097: *2043 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/06/02 15:35:48 [error] 1099#1099: *2057 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 53.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-53_80/", host: "53.example.com" +2025/06/02 15:35:49 [error] 1114#1114: *2097 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:35:49 [error] 1115#1115: *2111 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:35:49 [error] 1117#1117: *2126 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 54.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-54_80/", host: "54.example.com" +2025/06/02 15:35:50 [error] 1132#1132: *2165 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:35:50 [error] 1133#1133: *2180 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:35:50 [error] 1135#1135: *2194 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 55.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-55_80/", host: "55.example.com" +2025/06/02 15:35:52 [error] 1152#1152: *2227 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:35:52 [error] 1153#1153: *2240 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:35:52 [error] 1155#1155: *2254 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 56.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-56_80/", host: "56.example.com" +2025/06/02 15:35:53 [error] 1170#1170: *2293 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:35:53 [error] 1171#1171: *2306 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:35:53 [error] 1173#1173: *2320 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 57.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-57_80/", host: "57.example.com" +2025/06/02 15:35:54 [error] 1188#1188: *2363 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:35:54 [error] 1189#1189: *2375 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:35:54 [error] 1191#1191: *2386 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:35:54 [error] 1192#1192: *2400 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 58.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-58_80/", host: "58.example.com" +2025/06/02 15:35:55 [error] 1208#1208: *2429 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:35:56 [error] 1209#1209: *2441 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:35:56 [error] 1211#1211: *2456 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:35:56 [error] 1212#1212: *2470 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 59.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-59_80/", host: "59.example.com" +2025/06/02 15:35:57 [error] 1226#1226: *2503 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:35:57 [error] 1227#1227: *2517 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:35:57 [error] 1229#1229: *2532 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:35:58 [error] 1230#1230: *2544 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 60.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-60_80/", host: "60.example.com" +2025/06/02 15:35:59 [error] 1244#1244: *2576 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:35:59 [error] 1245#1245: *2590 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:35:59 [error] 1247#1247: *2600 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:35:59 [error] 1248#1248: *2614 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 61.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-61_80/", host: "61.example.com" +2025/06/02 15:36:01 [error] 1264#1264: *2650 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:36:01 [error] 1265#1265: *2664 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:36:01 [error] 1267#1267: *2680 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:36:01 [error] 1268#1268: *2692 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 62.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-62_80/", host: "62.example.com" +2025/06/02 15:36:03 [error] 1282#1282: *2726 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/06/02 15:36:04 [error] 1283#1283: *2740 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/06/02 15:36:04 [error] 1285#1285: *2753 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" +2025/06/02 15:36:04 [error] 1286#1286: *2767 no live upstreams while connecting to upstream, client: 34.82.94.226, server: 63.example.com, request: "GET / HTTP/1.1", upstream: "http://scale_backend-63_80/", host: "63.example.com" diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/ttr-oss.png b/tests/results/scale/2.0.0/TestScale_Listeners/ttr-oss.png new file mode 100644 index 0000000000..1e46255f88 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/ttr-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_Listeners/ttr-plus.png b/tests/results/scale/2.0.0/TestScale_Listeners/ttr-plus.png new file mode 100644 index 0000000000..19a9c34f0f Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_Listeners/ttr-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-oss.png b/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-oss.png new file mode 100644 index 0000000000..7aea4e65fb Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-plus.png b/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-plus.png new file mode 100644 index 0000000000..73a4997357 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_UpstreamServers/cpu-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-oss.png b/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-oss.png new file mode 100644 index 0000000000..ce3a80a2c2 Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-oss.png differ diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-plus.png b/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-plus.png new file mode 100644 index 0000000000..f4b12ebb9c Binary files /dev/null and b/tests/results/scale/2.0.0/TestScale_UpstreamServers/memory-plus.png differ diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-oss.log b/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-oss.log new file mode 100644 index 0000000000..b012d11e51 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-oss.log @@ -0,0 +1,2 @@ +{"level":"debug","ts":"2025-06-02T15:50:42Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gatewayclasses.gateway.networking.k8s.io \"nginx\": the object has been modified; please apply your changes to the latest version and try again","namespace":"","name":"nginx","kind":"GatewayClass"} +{"level":"debug","ts":"2025-06-02T15:51:09Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-plus.log b/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-plus.log new file mode 100644 index 0000000000..ba30848359 --- /dev/null +++ b/tests/results/scale/2.0.0/TestScale_UpstreamServers/ngf-plus.log @@ -0,0 +1 @@ +{"level":"debug","ts":"2025-06-02T15:52:39Z","logger":"statusUpdater","msg":"Encountered error updating status","error":"Operation cannot be fulfilled on gateways.gateway.networking.k8s.io \"gateway\": the object has been modified; please apply your changes to the latest version and try again","namespace":"scale","name":"gateway","kind":"Gateway"} diff --git a/tests/results/zero-downtime-scale/2.0.0/2.0.0-oss.md b/tests/results/zero-downtime-scale/2.0.0/2.0.0-oss.md new file mode 100644 index 0000000000..cad007fc6b --- /dev/null +++ b/tests/results/zero-downtime-scale/2.0.0/2.0.0-oss.md @@ -0,0 +1,285 @@ +# Results + +## Test environment + +NGINX Plus: false + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Increased latency in requests. + +## One NGINX Pod runs per node Test Results + +### Scale Up Gradually + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.141ms +Latencies [min, mean, 50, 90, 95, 99, max] 694.688µs, 1.335ms, 1.2ms, 1.383ms, 1.489ms, 1.788ms, 213.102ms +Bytes In [total, mean] 4652952, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-affinity-https-oss.png](gradual-scale-up-affinity-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.107ms +Latencies [min, mean, 50, 90, 95, 99, max] 681.597µs, 1.24ms, 1.176ms, 1.341ms, 1.428ms, 1.746ms, 152.544ms +Bytes In [total, mean] 4832961, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-affinity-http-oss.png](gradual-scale-up-affinity-http-oss.png) + +### Scale Down Gradually + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 48000, 100.00, 100.00 +Duration [total, attack, wait] 8m0s, 8m0s, 1.482ms +Latencies [min, mean, 50, 90, 95, 99, max] 646.651µs, 1.223ms, 1.204ms, 1.425ms, 1.533ms, 1.799ms, 15.965ms +Bytes In [total, mean] 7732691, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:48000 +Error Set: +``` + +![gradual-scale-down-affinity-http-oss.png](gradual-scale-down-affinity-http-oss.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 48000, 100.00, 100.00 +Duration [total, attack, wait] 8m0s, 8m0s, 1.502ms +Latencies [min, mean, 50, 90, 95, 99, max] 692.557µs, 1.277ms, 1.226ms, 1.484ms, 1.597ms, 1.876ms, 214.566ms +Bytes In [total, mean] 7444787, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:48000 +Error Set: +``` + +![gradual-scale-down-affinity-https-oss.png](gradual-scale-down-affinity-https-oss.png) + +### Scale Up Abruptly + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.147ms +Latencies [min, mean, 50, 90, 95, 99, max] 701.455µs, 1.259ms, 1.211ms, 1.543ms, 1.649ms, 1.905ms, 6.534ms +Bytes In [total, mean] 1861234, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-up-affinity-https-oss.png](abrupt-scale-up-affinity-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.133ms +Latencies [min, mean, 50, 90, 95, 99, max] 678.456µs, 1.238ms, 1.207ms, 1.493ms, 1.601ms, 1.835ms, 7.908ms +Bytes In [total, mean] 1933243, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-up-affinity-http-oss.png](abrupt-scale-up-affinity-http-oss.png) + +### Scale Down Abruptly + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.252ms +Latencies [min, mean, 50, 90, 95, 99, max] 679.727µs, 1.367ms, 1.333ms, 1.595ms, 1.689ms, 1.942ms, 205.763ms +Bytes In [total, mean] 1861231, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-affinity-https-oss.png](abrupt-scale-down-affinity-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.173ms +Latencies [min, mean, 50, 90, 95, 99, max] 747.649µs, 1.335ms, 1.318ms, 1.548ms, 1.631ms, 1.882ms, 20.583ms +Bytes In [total, mean] 1933152, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-affinity-http-oss.png](abrupt-scale-down-affinity-http-oss.png) + +## Multiple NGINX Pods run per node Test Results + +### Scale Up Gradually + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 822.54µs +Latencies [min, mean, 50, 90, 95, 99, max] 632.932µs, 1.174ms, 1.161ms, 1.323ms, 1.394ms, 1.75ms, 219.038ms +Bytes In [total, mean] 4832958, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-http-oss.png](gradual-scale-up-http-oss.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 932.669µs +Latencies [min, mean, 50, 90, 95, 99, max] 695.876µs, 1.213ms, 1.177ms, 1.331ms, 1.397ms, 1.778ms, 215.163ms +Bytes In [total, mean] 4653003, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-https-oss.png](gradual-scale-up-https-oss.png) + +### Scale Down Gradually + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 96000, 100.00, 100.00 +Duration [total, attack, wait] 16m0s, 16m0s, 1.153ms +Latencies [min, mean, 50, 90, 95, 99, max] 653.358µs, 1.208ms, 1.191ms, 1.357ms, 1.437ms, 1.725ms, 208.482ms +Bytes In [total, mean] 14889785, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:96000 +Error Set: +``` + +![gradual-scale-down-https-oss.png](gradual-scale-down-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 96000, 100.00, 100.00 +Duration [total, attack, wait] 16m0s, 16m0s, 1.064ms +Latencies [min, mean, 50, 90, 95, 99, max] 644.11µs, 1.193ms, 1.183ms, 1.359ms, 1.436ms, 1.716ms, 208.817ms +Bytes In [total, mean] 15465607, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:96000 +Error Set: +``` + +![gradual-scale-down-http-oss.png](gradual-scale-down-http-oss.png) + +### Scale Up Abruptly + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.245ms +Latencies [min, mean, 50, 90, 95, 99, max] 687.053µs, 1.223ms, 1.201ms, 1.436ms, 1.546ms, 1.778ms, 5.903ms +Bytes In [total, mean] 1861203, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-up-https-oss.png](abrupt-scale-up-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.27ms +Latencies [min, mean, 50, 90, 95, 99, max] 684.231µs, 1.18ms, 1.169ms, 1.354ms, 1.439ms, 1.726ms, 7.767ms +Bytes In [total, mean] 1933221, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-up-http-oss.png](abrupt-scale-up-http-oss.png) + +### Scale Down Abruptly + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.361ms +Latencies [min, mean, 50, 90, 95, 99, max] 657.739µs, 1.241ms, 1.221ms, 1.383ms, 1.451ms, 1.661ms, 209.787ms +Bytes In [total, mean] 1861235, 155.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-https-oss.png](abrupt-scale-down-https-oss.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.356ms +Latencies [min, mean, 50, 90, 95, 99, max] 684.483µs, 1.211ms, 1.21ms, 1.368ms, 1.431ms, 1.618ms, 12.796ms +Bytes In [total, mean] 1933178, 161.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-http-oss.png](abrupt-scale-down-http-oss.png) diff --git a/tests/results/zero-downtime-scale/2.0.0/2.0.0-plus.md b/tests/results/zero-downtime-scale/2.0.0/2.0.0-plus.md new file mode 100644 index 0000000000..dcffbbcf7c --- /dev/null +++ b/tests/results/zero-downtime-scale/2.0.0/2.0.0-plus.md @@ -0,0 +1,294 @@ +# Results + +## Test environment + +NGINX Plus: true + +NGINX Gateway Fabric: + +- Commit: f06b10da4a1754e253a58a31d2eff1bd404e33cd +- Date: 2025-06-02T15:20:05Z +- Dirty: false + +GKE Cluster: + +- Node count: 12 +- k8s version: v1.32.4-gke.1106006 +- vCPUs per node: 16 +- RAM per node: 65851340Ki +- Max pods per node: 110 +- Zone: us-west1-b +- Instance Type: n2d-standard-16 + +## Summary: + +- Increased latency in requests. +- Quite a few 502 errors, could be due to load balancer issues for this test run. + +## One NGINX Pod runs per node Test Results + +### Scale Up Gradually + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.239ms +Latencies [min, mean, 50, 90, 95, 99, max] 605.991µs, 1.15ms, 1.13ms, 1.331ms, 1.417ms, 1.722ms, 13.752ms +Bytes In [total, mean] 4622957, 154.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-affinity-https-plus.png](gradual-scale-up-affinity-https-plus.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.117ms +Latencies [min, mean, 50, 90, 95, 99, max] 604.088µs, 1.133ms, 1.113ms, 1.311ms, 1.385ms, 1.696ms, 206.07ms +Bytes In [total, mean] 4805954, 160.20 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-affinity-http-plus.png](gradual-scale-up-affinity-http-plus.png) + +### Scale Down Gradually + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 48000, 100.00, 100.00 +Duration [total, attack, wait] 8m0s, 8m0s, 1.358ms +Latencies [min, mean, 50, 90, 95, 99, max] 616.744µs, 1.166ms, 1.153ms, 1.344ms, 1.416ms, 1.668ms, 16.868ms +Bytes In [total, mean] 7396614, 154.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:48000 +Error Set: +``` + +![gradual-scale-down-affinity-https-plus.png](gradual-scale-down-affinity-https-plus.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 48000, 100.00, 100.00 +Duration [total, attack, wait] 8m0s, 8m0s, 1.306ms +Latencies [min, mean, 50, 90, 95, 99, max] 584.562µs, 1.143ms, 1.127ms, 1.306ms, 1.368ms, 1.614ms, 215.829ms +Bytes In [total, mean] 7689634, 160.20 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:48000 +Error Set: +``` + +![gradual-scale-down-affinity-http-plus.png](gradual-scale-down-affinity-http-plus.png) + +### Scale Up Abruptly + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 66.67 +Duration [total, attack, wait] 2m0s, 2m0s, 510.001µs +Latencies [min, mean, 50, 90, 95, 99, max] 401.714µs, 935.722µs, 1.033ms, 1.245ms, 1.304ms, 1.452ms, 7.179ms +Bytes In [total, mean] 1880825, 156.74 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 66.67% +Status Codes [code:count] 0:5 200:8000 502:3995 +Error Set: +Get "http://cafe.example.com/coffee": dial tcp 0.0.0.0:0->redacted:80: connect: connection refused +502 Bad Gateway +``` + +![abrupt-scale-up-affinity-http-plus.png](abrupt-scale-up-affinity-http-plus.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.00 +Duration [total, attack, wait] 2m0s, 2m0s, 876.584µs +Latencies [min, mean, 50, 90, 95, 99, max] 601.132µs, 1.175ms, 1.165ms, 1.358ms, 1.422ms, 1.638ms, 12.009ms +Bytes In [total, mean] 1849023, 154.09 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 99.99% +Status Codes [code:count] 0:1 200:11999 +Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->redacted:443: connect: connection refused +``` + +![abrupt-scale-up-affinity-https-plus.png](abrupt-scale-up-affinity-https-plus.png) + +### Scale Down Abruptly + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 75.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.354ms +Latencies [min, mean, 50, 90, 95, 99, max] 362.267µs, 1.05ms, 1.141ms, 1.365ms, 1.435ms, 1.605ms, 17.379ms +Bytes In [total, mean] 1891781, 157.65 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 75.00% +Status Codes [code:count] 200:9000 502:3000 +Error Set: +502 Bad Gateway +``` + +![abrupt-scale-down-affinity-http-plus.png](abrupt-scale-down-affinity-http-plus.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.269ms +Latencies [min, mean, 50, 90, 95, 99, max] 600.937µs, 1.218ms, 1.206ms, 1.405ms, 1.476ms, 1.685ms, 8.108ms +Bytes In [total, mean] 1849128, 154.09 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-affinity-https-plus.png](abrupt-scale-down-affinity-https-plus.png) + +## Multiple NGINX Pods run per node Test Results + +### Scale Up Gradually + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.181ms +Latencies [min, mean, 50, 90, 95, 99, max] 622.202µs, 1.144ms, 1.132ms, 1.304ms, 1.369ms, 1.648ms, 211.565ms +Bytes In [total, mean] 4805917, 160.20 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-http-plus.png](gradual-scale-up-http-plus.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 30000, 100.00, 100.00 +Duration [total, attack, wait] 5m0s, 5m0s, 1.117ms +Latencies [min, mean, 50, 90, 95, 99, max] 644.069µs, 1.174ms, 1.163ms, 1.339ms, 1.409ms, 1.704ms, 16.884ms +Bytes In [total, mean] 4623044, 154.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:30000 +Error Set: +``` + +![gradual-scale-up-https-plus.png](gradual-scale-up-https-plus.png) + +### Scale Down Gradually + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 96000, 100.00, 100.00 +Duration [total, attack, wait] 16m0s, 16m0s, 982.209µs +Latencies [min, mean, 50, 90, 95, 99, max] 599.14µs, 1.181ms, 1.156ms, 1.349ms, 1.427ms, 1.7ms, 215.457ms +Bytes In [total, mean] 14793636, 154.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:96000 +Error Set: +``` + +![gradual-scale-down-https-plus.png](gradual-scale-down-https-plus.png) + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 96000, 100.00, 100.00 +Duration [total, attack, wait] 16m0s, 16m0s, 1.113ms +Latencies [min, mean, 50, 90, 95, 99, max] 583.608µs, 1.133ms, 1.123ms, 1.301ms, 1.368ms, 1.621ms, 206.955ms +Bytes In [total, mean] 15379258, 160.20 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:96000 +Error Set: +``` + +![gradual-scale-down-http-plus.png](gradual-scale-down-http-plus.png) + +### Scale Up Abruptly + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 74.91 +Duration [total, attack, wait] 2m0s, 2m0s, 706.902µs +Latencies [min, mean, 50, 90, 95, 99, max] 363.273µs, 980.321µs, 1.06ms, 1.271ms, 1.338ms, 1.561ms, 12.623ms +Bytes In [total, mean] 1889876, 157.49 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 74.90% +Status Codes [code:count] 0:12 200:8988 502:3000 +Error Set: +Get "http://cafe.example.com/coffee": dial tcp 0.0.0.0:0->redacted:80: connect: connection refused +502 Bad Gateway +``` + +![abrupt-scale-up-http-plus.png](abrupt-scale-up-http-plus.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 99.92 +Duration [total, attack, wait] 2m0s, 2m0s, 1.077ms +Latencies [min, mean, 50, 90, 95, 99, max] 540.182µs, 1.147ms, 1.138ms, 1.32ms, 1.39ms, 1.628ms, 12.628ms +Bytes In [total, mean] 1847648, 153.97 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 99.92% +Status Codes [code:count] 0:10 200:11990 +Error Set: +Get "https://cafe.example.com/tea": dial tcp 0.0.0.0:0->redacted:443: connect: connection refused +``` + +![abrupt-scale-up-https-plus.png](abrupt-scale-up-https-plus.png) + +### Scale Down Abruptly + +#### Test: Send http /coffee traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 91.67 +Duration [total, attack, wait] 2m0s, 2m0s, 1.117ms +Latencies [min, mean, 50, 90, 95, 99, max] 401.425µs, 1.104ms, 1.117ms, 1.3ms, 1.364ms, 1.598ms, 217.29ms +Bytes In [total, mean] 1912123, 159.34 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 91.67% +Status Codes [code:count] 200:11000 502:1000 +Error Set: +502 Bad Gateway +``` + +![abrupt-scale-down-http-plus.png](abrupt-scale-down-http-plus.png) + +#### Test: Send https /tea traffic + +```text +Requests [total, rate, throughput] 12000, 100.01, 100.01 +Duration [total, attack, wait] 2m0s, 2m0s, 1.072ms +Latencies [min, mean, 50, 90, 95, 99, max] 627.313µs, 1.143ms, 1.142ms, 1.308ms, 1.371ms, 1.519ms, 8.844ms +Bytes In [total, mean] 1849201, 154.10 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 100.00% +Status Codes [code:count] 200:12000 +Error Set: +``` + +![abrupt-scale-down-https-plus.png](abrupt-scale-down-https-plus.png) diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-oss.png new file mode 100644 index 0000000000..084a7d8904 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-plus.png new file mode 100644 index 0000000000..d4e85c381b Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-oss.png new file mode 100644 index 0000000000..084a7d8904 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-plus.png new file mode 100644 index 0000000000..e3a7206f5b Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-oss.png new file mode 100644 index 0000000000..ef8aee1a90 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-plus.png new file mode 100644 index 0000000000..bdbd2f9ed7 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-oss.png new file mode 100644 index 0000000000..ef8aee1a90 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-plus.png new file mode 100644 index 0000000000..51c4546e3d Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-down-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-oss.png new file mode 100644 index 0000000000..372f5e6435 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-plus.png new file mode 100644 index 0000000000..d7fe35d51a Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-oss.png new file mode 100644 index 0000000000..372f5e6435 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-plus.png new file mode 100644 index 0000000000..d105077e13 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-oss.png new file mode 100644 index 0000000000..6e6bb0f14f Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-plus.png new file mode 100644 index 0000000000..4a3ce144bc Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-oss.png new file mode 100644 index 0000000000..6e6bb0f14f Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-plus.png new file mode 100644 index 0000000000..0808fa4ca8 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/abrupt-scale-up-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-oss.png new file mode 100644 index 0000000000..fa9a228d98 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-plus.png new file mode 100644 index 0000000000..c9358a6f17 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-oss.png new file mode 100644 index 0000000000..fa9a228d98 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-plus.png new file mode 100644 index 0000000000..c9358a6f17 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-oss.png new file mode 100644 index 0000000000..56a68e1429 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-plus.png new file mode 100644 index 0000000000..8eb9d8fe44 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-oss.png new file mode 100644 index 0000000000..56a68e1429 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-plus.png new file mode 100644 index 0000000000..8eb9d8fe44 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-down-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-oss.png new file mode 100644 index 0000000000..6db150f3b9 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-plus.png new file mode 100644 index 0000000000..45a6a1b33f Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-oss.png new file mode 100644 index 0000000000..6db150f3b9 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-plus.png new file mode 100644 index 0000000000..45a6a1b33f Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-affinity-https-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-oss.png new file mode 100644 index 0000000000..2eb1e82849 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-plus.png new file mode 100644 index 0000000000..a99749351a Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-http-plus.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-oss.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-oss.png new file mode 100644 index 0000000000..2eb1e82849 Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-oss.png differ diff --git a/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-plus.png b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-plus.png new file mode 100644 index 0000000000..a99749351a Binary files /dev/null and b/tests/results/zero-downtime-scale/2.0.0/gradual-scale-up-https-plus.png differ