Merge branch 'main' into fix-boto

pillar/base/haproxy.sls

@@ -75,6 +75,7 @@ haproxy:
         - {{ config.server_name }}
       verify_host: bugs.psf.io
       check: "HEAD / HTTP/1.1\\r\\nHost:\\ {{ config.server_name }}"
+      rate_limit: {{ config.get('rate_limit', 100) }}
     {% endfor %}
 
     moin:

salt/bugs/config/nginx.conf.jinja

@@ -3,7 +3,7 @@ log_format timed_combined_{{ tracker }} '$remote_addr - $remote_user [$time_loca
     '"$http_referer" "$http_user_agent" '
     '$request_time $upstream_response_time $pipe';
 
-limit_req_zone $binary_remote_addr zone=limit-{{ tracker }}:10m rate=1r/s;
+limit_req_zone $binary_remote_addr zone=limit-{{ tracker }}:10m rate=5r/s;
 
 
 upstream tracker-{{ tracker }} {
@@ -53,7 +53,7 @@ server {
   }
 
   location / {
-    limit_req zone=limit-{{ tracker }} burst=5 nodelay;
+    limit_req zone=limit-{{ tracker }} burst=10 nodelay;
     proxy_pass http://tracker-{{ tracker }}/;
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;

salt/haproxy/config/haproxy.cfg.jinja

@@ -52,7 +52,6 @@ global
     # Lower the amount of space we reserve for header rewriting
     tune.maxrewrite 1024
 
-
 defaults
     log     global
 
@@ -84,7 +83,14 @@ defaults
     errorfile 504 /etc/haproxy/errors/504.http
 
 
-listen tls:
+frontend main
+    # HTTP Binds
+    bind :20000
+    bind :20001 accept-proxy
+    bind 0.0.0.0:80
+    bind :::80
+
+    # HTTPS Binds
     # Advertise http/1.1 over NPN to enable TLS False Start
     bind :20003 ssl alpn h2,http/1.1 crt star.python.org.pem
     bind :20005 ssl alpn h2,http/1.1 crt star.pypa.io.pem
@@ -96,26 +102,23 @@ listen tls:
     bind :20010 ssl alpn h2,http/1.1 crt star.python.org.pem crt star.pypa.io.pem crt star.pyfound.org.pem crt speed.pypy.org.pem crt www.pycon.org.pem crt jython.org.pem crt salt-public.psf.io.pem crt planetpython.org.pem crt bugs.python.org.pem
     bind :20011 accept-proxy ssl alpn h2,http/1.1 crt star.python.org.pem crt star.pypa.io.pem crt star.pyfound.org.pem crt speed.pypy.org.pem crt www.pycon.org.pem crt jython.org.pem crt salt-public.psf.io.pem crt planetpython.org.pem crt bugs.python.org.pem
 
-    mode http
-
-    option httplog
-
-    # Capture the user agent in the log
-    capture request header User-Agent len 512
-    # Capture X-Forwarded-For in the log
-    capture request header X-Forwarded-For len 64
-
-    http-request set-header X-Client-IP %[src]
-
-    server default 127.0.0.1:19001
-
-
-frontend main
-    bind :20000
-    bind :20001 accept-proxy
-    bind 0.0.0.0:80
-    bind :::80
-    bind 127.0.0.1:19001  # This is our TLS socket.
+    # Define a stick table for all services
+    stick-table type ipv6 size 100k expire 30s store http_req_rate(10s)
+    # Track all requests using a single counter
+    # We could use the 3 available (sc0,1,2) to maybe tier requests
+    # into say <=100, 101-500, >= 501 if we needed to?
+    http-request track-sc0 src
+    # then create the ACL for services in haproxy.sls that have a 'rate_limit' key,
+    # constrained to the host header using the domain key in haproxy.sls
+    # then adds a rule to deny via HTTP 429 if the respective ACL is matched and the stick table http request rate
+    # is higher than the 'rate_limit' from haproxy.sls pillar date
+    {%- for service, config in haproxy.services.items() %}
+    {%- if config.get('rate_limit') %}
+    # Rate limit config for {{ service }}
+    acl is_{{ service }} hdr(host) -i {% for domain in config.domains %}{{ domain }} {% endfor %}
+    http-request deny deny_status 429 if is_{{ service }} { sc_http_req_rate(0) gt {{ config.rate_limit }} }
+    {%- endif %}
+    {%- endfor %}
 
     # Custom logging format, this is the same as the normal "httplog" in
     # HAProxy except information about the TLS connection is included.
@@ -130,17 +133,17 @@ frontend main
     use_backend letsencrypt-well-known if letsencrypt-well-known-acl
 
     # Determine if this request has TLS on the client side or not.
-    acl is_tls dst_port 19001
+    acl is_tls ssl_fc
 
     acl our_domains hdr(host) -i -f /etc/haproxy/our_domains
 
     # Deny requests that are not served from this host
     http-request deny if !our_domains !letsencrypt-well-known-acl
 
+    http-request set-header X-Client-IP %[src]
     # Tell the backend servers whether this request is being served via TLS or
     # not. This should pretty much *always* be yes since we unconditionally
     # redirect to HTTPS in HAProxy.
-    http-request set-header X-Client-IP %[src] if !is_tls
     http-request set-header X-Forwarded-Proto https if is_tls
     http-request set-header X-Forwarded-Proto http  if !is_tls
 

salt/haproxy/init.sls

@@ -117,17 +117,6 @@ haproxy-ocsp:
 {% endif %}
 
 
-/etc/nginx/sites.d/http2.conf:
-  file.managed:
-    - source: salt://haproxy/config/nginx-http2.conf.jinja
-    - template: jinja
-    - user: root
-    - group: root
-    - mode: "0644"
-    - require:
-      - file: /etc/nginx/sites.d/
-
-
 /etc/nginx/sites.d/redirect.conf:
   file.managed:
     - source: salt://haproxy/config/nginx-redirect.conf.jinja