Skip to content

Add router support for wildcard domains (*.foo.com) #11201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-bot merged 2 commits into from
Oct 21, 2016
Merged

Add router support for wildcard domains (*.foo.com) #11201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ff7565
Fix an issue with route ordering: it was possible for a newer route t…
ramr Oct 14, 2016
7ab4f0f
o Split out cert list and use commit from PR 11217
ramr Oct 13, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions contrib/completions/bash/openshift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19225,6 +19225,10 @@ _openshift_infra_f5-router()
flags_with_completion=()
flags_completion=()

flags+=("--allow-wildcard-routes")
local_nonpersistent_flags+=("--allow-wildcard-routes")
flags+=("--allowed-domains=")
local_nonpersistent_flags+=("--allowed-domains=")
flags+=("--as=")
local_nonpersistent_flags+=("--as=")
flags+=("--certificate-authority=")
Expand All @@ -19247,6 +19251,8 @@ _openshift_infra_f5-router()
local_nonpersistent_flags+=("--config=")
flags+=("--context=")
local_nonpersistent_flags+=("--context=")
flags+=("--denied-domains=")
local_nonpersistent_flags+=("--denied-domains=")
flags+=("--f5-host=")
local_nonpersistent_flags+=("--f5-host=")
flags+=("--f5-http-vserver=")
Expand Down Expand Up @@ -19360,6 +19366,10 @@ _openshift_infra_router()
flags_with_completion=()
flags_completion=()

flags+=("--allow-wildcard-routes")
local_nonpersistent_flags+=("--allow-wildcard-routes")
flags+=("--allowed-domains=")
local_nonpersistent_flags+=("--allowed-domains=")
flags+=("--as=")
local_nonpersistent_flags+=("--as=")
flags+=("--certificate-authority=")
Expand Down Expand Up @@ -19388,6 +19398,8 @@ _openshift_infra_router()
local_nonpersistent_flags+=("--default-certificate-dir=")
flags+=("--default-certificate-path=")
local_nonpersistent_flags+=("--default-certificate-path=")
flags+=("--denied-domains=")
local_nonpersistent_flags+=("--denied-domains=")
flags+=("--extended-validation")
local_nonpersistent_flags+=("--extended-validation")
flags+=("--fields=")
Expand Down
12 changes: 12 additions & 0 deletions contrib/completions/zsh/openshift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19386,6 +19386,10 @@ _openshift_infra_f5-router()
flags_with_completion=()
flags_completion=()

flags+=("--allow-wildcard-routes")
local_nonpersistent_flags+=("--allow-wildcard-routes")
flags+=("--allowed-domains=")
local_nonpersistent_flags+=("--allowed-domains=")
flags+=("--as=")
local_nonpersistent_flags+=("--as=")
flags+=("--certificate-authority=")
Expand All @@ -19408,6 +19412,8 @@ _openshift_infra_f5-router()
local_nonpersistent_flags+=("--config=")
flags+=("--context=")
local_nonpersistent_flags+=("--context=")
flags+=("--denied-domains=")
local_nonpersistent_flags+=("--denied-domains=")
flags+=("--f5-host=")
local_nonpersistent_flags+=("--f5-host=")
flags+=("--f5-http-vserver=")
Expand Down Expand Up @@ -19521,6 +19527,10 @@ _openshift_infra_router()
flags_with_completion=()
flags_completion=()

flags+=("--allow-wildcard-routes")
local_nonpersistent_flags+=("--allow-wildcard-routes")
flags+=("--allowed-domains=")
local_nonpersistent_flags+=("--allowed-domains=")
flags+=("--as=")
local_nonpersistent_flags+=("--as=")
flags+=("--certificate-authority=")
Expand Down Expand Up @@ -19549,6 +19559,8 @@ _openshift_infra_router()
local_nonpersistent_flags+=("--default-certificate-dir=")
flags+=("--default-certificate-path=")
local_nonpersistent_flags+=("--default-certificate-path=")
flags+=("--denied-domains=")
local_nonpersistent_flags+=("--denied-domains=")
flags+=("--extended-validation")
local_nonpersistent_flags+=("--extended-validation")
flags+=("--fields=")
Expand Down
12 changes: 12 additions & 0 deletions docs/man/man1/openshift-infra-f5-router.1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ that you must have a cluster\-wide administrative role to view all namespaces.


.SH OPTIONS
.PP
\fB\-\-allow\-wildcard\-routes\fP=false
Allow wildcard host names for routes

.PP
\fB\-\-allowed\-domains\fP=[]
List of comma separated domains to allow in routes. If specified, only the domains in this list will be allowed routes. Note that domains in the denied list take precedence over the ones in the allowed list

.PP
\fB\-\-api\-version\fP=""
DEPRECATED: The API version to use when talking to the server
Expand Down Expand Up @@ -58,6 +66,10 @@ that you must have a cluster\-wide administrative role to view all namespaces.
\fB\-\-context\fP=""
The name of the kubeconfig context to use

.PP
\fB\-\-denied\-domains\fP=[]
List of comma separated domains to deny in routes

.PP
\fB\-\-f5\-host\fP=""
The host of F5 BIG\-IP's management interface
Expand Down
12 changes: 12 additions & 0 deletions docs/man/man1/openshift-infra-router.1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ that you must have a cluster\-wide administrative role to view all namespaces.


.SH OPTIONS
.PP
\fB\-\-allow\-wildcard\-routes\fP=false
Allow wildcard host names for routes

.PP
\fB\-\-allowed\-domains\fP=[]
List of comma separated domains to allow in routes. If specified, only the domains in this list will be allowed routes. Note that domains in the denied list take precedence over the ones in the allowed list

.PP
\fB\-\-api\-version\fP=""
DEPRECATED: The API version to use when talking to the server
Expand Down Expand Up @@ -78,6 +86,10 @@ that you must have a cluster\-wide administrative role to view all namespaces.
\fB\-\-default\-certificate\-path\fP=""
A path to default certificate to use for routes that don't expose a TLS server cert; in PEM format

.PP
\fB\-\-denied\-domains\fP=[]
List of comma separated domains to deny in routes

.PP
\fB\-\-extended\-validation\fP=true
If set, then an additional extended validation step is performed on all routes admitted in by this router. Defaults to true and enables the extended validation checks.
Expand Down
2 changes: 1 addition & 1 deletion images/router/haproxy/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ RUN INSTALL_PKGS="haproxy" && \
yum clean all && \
mkdir -p /var/lib/haproxy/router/{certs,cacerts} && \
mkdir -p /var/lib/haproxy/{conf,run,bin,log} && \
touch /var/lib/haproxy/conf/{{os_http_be,os_edge_http_be,os_tcp_be,os_sni_passthrough,os_reencrypt,os_edge_http_expose,os_edge_http_redirect,cert_config}.map,haproxy.config} && \
touch /var/lib/haproxy/conf/{{os_http_be,os_edge_http_be,os_tcp_be,os_sni_passthrough,os_reencrypt,os_edge_http_expose,os_edge_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \
chmod -R 777 /var && \
setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy

Expand Down
112 changes: 107 additions & 5 deletions images/router/haproxy/conf/haproxy-config.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,15 +100,36 @@ frontend public
acl secure_redirect base,map_beg(/var/lib/haproxy/conf/os_edge_http_redirect.map) -m found
redirect scheme https if secure_redirect

{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
Copy link
Contributor

@bmeng bmeng Oct 9, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will also meet this condition when the value is 1?

Copy link
Contributor Author

@ramr ramr Oct 10, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could do a 1 in that set as well if need be but true|TRUE follows the other parameters in this file.

Copy link
Contributor

@knobunc knobunc Oct 18, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, let's just support true for now. The SYN eater I added got it backwards and added '1' so I fixed that later and support 'true' and '1', but 'true' is preferred.

# Check for wildcard domains with redirected http routes.
acl wildcard_domain hdr(host),map_reg(/var/lib/haproxy/conf/os_wildcard_domain.map) -m found

acl wildcard_secure_redirect base,map_reg(/var/lib/haproxy/conf/os_edge_http_redirect.map) -m found
redirect scheme https if wildcard_domain wildcard_secure_redirect

{{ end }}

# Check if it is an edge route exposed insecurely.
acl edge_http_expose base,map_beg(/var/lib/haproxy/conf/os_edge_http_expose.map) -m found
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_expose.map)] if edge_http_expose

# map to http backend
# Search from most specific to general path (host case).
acl http_backend base,map_beg(/var/lib/haproxy/conf/os_http_be.map) -m found
use_backend be_http_%[base,map_beg(/var/lib/haproxy/conf/os_http_be.map)] if http_backend

{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
# Check for wildcard domains with exposed http routes.
acl wildcard_edge_http_expose base,map_reg(/var/lib/haproxy/conf/os_edge_http_expose.map) -m found
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_expose.map)] if wildcard_domain wildcard_edge_http_expose

# map to http backend
# Search from most specific to general path (host case).
# Note: If no match, haproxy uses the default_backend, no other
# use_backend directives below this will be processed.
use_backend be_http_%[base,map_beg(/var/lib/haproxy/conf/os_http_be.map)]
use_backend be_http_%[base,map_reg(/var/lib/haproxy/conf/os_http_be.map)] if wildcard_domain

{{ end }}

default_backend openshift_default

Expand All @@ -125,6 +146,15 @@ frontend public_ssl
acl sni_passthrough req.ssl_sni,map(/var/lib/haproxy/conf/os_sni_passthrough.map) -m found
use_backend be_tcp_%[req.ssl_sni,map(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_passthrough

{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
# Check for wildcard domains with passthrough.
acl sni_wildcard_domain req.ssl_sni,map_reg(/var/lib/haproxy/conf/os_wildcard_domain.map) -m found

acl sni_wildcard_passthrough req.ssl_sni,map_reg(/var/lib/haproxy/conf/os_sni_passthrough.map) -m found
use_backend be_tcp_%[req.ssl_sni,map_reg(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_wildcard_domain sni_wildcard_passthrough

{{ end }}

# if the route is SNI and NOT passthrough enter the termination flow
use_backend be_sni if sni

Expand Down Expand Up @@ -160,11 +190,25 @@ frontend fe_sni
# Search from most specific to general path (host case).
use_backend be_secure_%[base,map_beg(/var/lib/haproxy/conf/os_reencrypt.map)] if reencrypt

# map to http backend
# Search from most specific to general path (host case).
acl http_backend base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map) -m found
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map)] if http_backend

{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
# Check for wildcard domains with redirected or exposed http routes.
acl sni_wildcard_domain hdr(host),map_reg(/var/lib/haproxy/conf/os_wildcard_domain.map) -m found

acl wildcard_reencrypt base,map_reg(/var/lib/haproxy/conf/os_reencrypt.map) -m found
use_backend be_secure_%[base,map_reg(/var/lib/haproxy/conf/os_reencrypt.map)] if sni_wildcard_domain wildcard_reencrypt

# map to http backend
# Search from most specific to general path (host case).
# Note: If no match, haproxy uses the default_backend, no other
# use_backend directives below this will be processed.
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map)]
use_backend be_edge_http_%[base,map_reg(/var/lib/haproxy/conf/os_edge_http_be.map)] if sni_wildcard_domain

{{ end }}

default_backend openshift_default

Expand Down Expand Up @@ -197,11 +241,24 @@ frontend fe_no_sni
# Search from most specific to general path (host case).
use_backend be_secure_%[base,map_beg(/var/lib/haproxy/conf/os_reencrypt.map)] if reencrypt

# map to http backend
# Search from most specific to general path (host case).
acl edge_http_backend base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map) -m found
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map)] if edge_http_backend

{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
Copy link
Contributor

@smarterclayton smarterclayton Oct 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What performance impact does this have on a heavily loaded router? I assume host_wildcard_domain is expensive?

Copy link
Contributor Author

@ramr ramr Oct 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assumed it would be a wee bit more expensive because its doing a regexp compare so there is some cost to matching against a pattern. But it is in a if block so is only enabled if the router allows wildcard routes.

The strange part was just did a quick and dirty test with ab and I see different results - removed https out of the mix as there is overhead benchmarking w/ cert checks etc and this is what I see:

For a wildcard route that allows http requests and returns a header dump:
time ab -n 10000 -c 50 -H "Host: www2.open.header.test" http://127.0.0.1/
50% of the requests finished in 13 ms, 95% finished in 19ms, and 99% in 23 ms - longest was 27ms. Elapsed time = 0m2.777s

For a non-wildcard route:
time ab -n 10000 -c 50 -H "Host: allow-http.header.test" http://127.0.0.1/
50% of the requests took the same time 13 ms, 95% took 21 ms , 99% took 25 ms and the longest one took 28ms. Elapsed time: 0m2.855s

So the wildcard checks are actually a wee bit faster thanks to a compiled regexp. Of course the performance test is not a real-world test but it still does have some merit and the results are not to say the least interesting.

Copy link
Contributor

@smarterclayton smarterclayton Oct 21, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the plan to enable this in any of the online environments?

Copy link
Member

@eparis eparis Oct 21, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At this moment I do not know. We have never been asked by online for this. However, online turns on just about everything they learn about, so likely we need to make sure they understand the implications.

Copy link
Contributor

@liggitt liggitt Oct 21, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would expect the expense to show up when switching among lots of routes, not repeatedly hitting a router serving few routes.

acl host_wildcard_domain req.ssl_sni,map_reg(/var/lib/haproxy/conf/os_wildcard_domain.map) -m found

acl host_reencrypt base,map_reg(/var/lib/haproxy/conf/os_reencrypt.map) -m found
use_backend be_secure_%[base,map_reg(/var/lib/haproxy/conf/os_reencrypt.map)] if host_wildcard_domain host_reencrypt

# map to http backend
# Search from most specific to general path (host case).
# Note: If no match, haproxy uses the default_backend, no other
# use_backend directives below this will be processed.
use_backend be_edge_http_%[base,map_beg(/var/lib/haproxy/conf/os_edge_http_be.map)]
use_backend be_edge_http_%[base,map_reg(/var/lib/haproxy/conf/os_edge_http_be.map)] if host_wildcard_domain

{{ end }}

default_backend openshift_default

Expand Down Expand Up @@ -321,8 +378,8 @@ backend be_edge_http_{{$cfgIdx}}
{{ end }}{{/* end iterate over services */}}
{{ end }}{{/* end if tls==edge/none */}}

# Secure backend, pass through
{{ if eq $cfg.TLSTermination "passthrough" }}
# Secure backend, pass through
backend be_tcp_{{$cfgIdx}}
{{ if ne (env "ROUTER_SYSLOG_ADDRESS" "") ""}}
option tcplog
Expand Down Expand Up @@ -385,8 +442,8 @@ backend be_tcp_{{$cfgIdx}}
{{ end }}{{/* end iterate over services*/}}
{{ end }}{{/*end tls==passthrough*/}}

# Secure backend which requires re-encryption
{{ if eq $cfg.TLSTermination "reencrypt" }}
# Secure backend which requires re-encryption
backend be_secure_{{$cfgIdx}}
mode http
option redispatch
Expand Down Expand Up @@ -462,14 +519,35 @@ backend be_secure_{{$cfgIdx}}
{{ end }}{{/* end haproxy config template */}}

{{/*--------------------------------- END OF HAPROXY CONFIG, BELOW ARE MAPPING FILES ------------------------*/}}
{{/*
os_wildcard_domain.map: contains a mapping of wildcard hosts for a
[sub]domain regexps. This map is used to check if
a host matches a [sub]domain with has wildcard support.
*/}}
{{ define "/var/lib/haproxy/conf/os_wildcard_domain.map" }}
{{ if matchPattern "true|TRUE" (env "ROUTER_ALLOW_WILDCARD_ROUTES" "")}}
{{ range $idx, $cfg := .State }}
{{ if ne $cfg.Host ""}}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host "" true}} 1
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end if router allows wildcard routes */}}
{{ end }}{{/* end wildcard domain map template */}}

{{/*
os_http_be.map: contains a mapping of www.example.com -> <service name>. This map is used to discover the correct backend
by attaching a prefix (be_http_) by use_backend statements if acls are matched.
*/}}
{{ define "/var/lib/haproxy/conf/os_http_be.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "")}}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host $cfg.Path false}} {{$idx}}
{{ else }}
{{$cfg.Host}}{{$cfg.Path}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end http host map template */}}
Expand All @@ -481,7 +559,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_edge_http_be.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "edge")}}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host $cfg.Path false}} {{$idx}}
{{ else }}
{{$cfg.Host}}{{$cfg.Path}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end edge http host map template */}}
Expand All @@ -494,7 +576,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_edge_http_expose.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (ne $cfg.Host "") (and (eq $cfg.TLSTermination "edge") (eq $cfg.InsecureEdgeTerminationPolicy "Allow"))}}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host $cfg.Path false}} {{$idx}}
{{ else }}
{{$cfg.Host}}{{$cfg.Path}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end edge insecure expose http host map template */}}
Expand All @@ -507,7 +593,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_edge_http_redirect.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (ne $cfg.Host "") (and (eq $cfg.TLSTermination "edge") (eq $cfg.InsecureEdgeTerminationPolicy "Redirect"))}}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host $cfg.Path false}} {{$idx}}
{{ else }}
{{$cfg.Host}}{{$cfg.Path}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end edge insecure redirect http host map template */}}
Expand All @@ -520,7 +610,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_tcp_be.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (eq $cfg.Path "") (and (ne $cfg.Host "") (or (eq $cfg.TLSTermination "passthrough") (eq $cfg.TLSTermination "reencrypt"))) }}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host "" true}} {{$idx}}
{{ else }}
{{$cfg.Host}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end tcp host map template */}}
Expand All @@ -532,7 +626,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_sni_passthrough.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (eq $cfg.Path "") (eq $cfg.TLSTermination "passthrough") }}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host "" true}} {{$idx}}
{{ else }}
{{$cfg.Host}} 1
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end sni passthrough map template */}}
Expand All @@ -545,7 +643,11 @@ backend be_secure_{{$cfgIdx}}
{{ define "/var/lib/haproxy/conf/os_reencrypt.map" }}
{{ range $idx, $cfg := .State }}
{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "reencrypt") }}
{{ if $cfg.IsWildcard }}
{{genDomainWildcardRegexp $cfg.Host $cfg.Path false}} {{$idx}}
{{ else }}
{{$cfg.Host}}{{$cfg.Path}} {{$idx}}
{{ end }}
{{ end }}
{{ end }}
{{ end }}{{/* end reencrypt map template */}}
Expand Down
Loading