Skip to content

Commit 8fdde07

Browse files
LorcanMcVeighLorcanMcVeigh
authored
Add snippet feature to TransportServer (#1413)
1 parent b6b2807 commit 8fdde07

17 files changed

+222
-39
lines changed

‎cmd/nginx-ingress/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,7 @@ func main() {
604604
templateExecutorV2, *nginxPlus, isWildcardEnabled, plusCollector, *enablePrometheusMetrics, latencyCollector, *enableLatencyMetrics)
605605
controllerNamespace := os.Getenv("POD_NAMESPACE")
606606

607-
transportServerValidator := cr_validation.NewTransportServerValidator(*enableTLSPassthrough)
607+
transportServerValidator := cr_validation.NewTransportServerValidator(*enableTLSPassthrough, *enableSnippets)
608608
virtualServerValidator := cr_validation.NewVirtualServerValidator(*nginxPlus)
609609

610610
lbcInput := k8s.NewLoadBalancerControllerInput{

‎deployments/common/crds-v1beta1/k8s.nginx.org_transportservers.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ spec:
6464
type: string
6565
protocol:
6666
type: string
67+
serverSnippets:
68+
type: string
6769
sessionParameters:
6870
description: SessionParameters defines session parameters.
6971
type: object

‎deployments/common/crds/k8s.nginx.org_transportservers.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ spec:
6363
type: string
6464
protocol:
6565
type: string
66+
serverSnippets:
67+
type: string
6668
sessionParameters:
6769
description: SessionParameters defines session parameters.
6870
type: object

‎deployments/helm-chart/crds/k8s.nginx.org_transportservers.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ spec:
6363
type: string
6464
protocol:
6565
type: string
66+
serverSnippets:
67+
type: string
6668
sessionParameters:
6769
description: SessionParameters defines session parameters.
6870
type: object

‎docs-web/configuration/transportserver-resource.md

+36
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ This document is the reference documentation for the TransportServer resource. T
1919
- [SessionParameters](#sessionparameters)
2020
- [Action](#action)
2121
- [Using TransportServer](#using-transportserver)
22+
- [Usings Snippets](#using-snippets)
2223
- [Validation](#validation)
2324
- [Structural Validation](#structural-validation)
2425
- [Comprehensive Validation](#comprehensive-validation)
@@ -370,6 +371,41 @@ secure-app 46sm
370371

371372
In the kubectl get and similar commands, you can also use the short name `ts` instead of `transportserver`.
372373

374+
### Using Snippets
375+
376+
Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure [access control](http://nginx.org/en/docs/stream/ngx_stream_access_module.html) in a TransportServer:
377+
378+
```yaml
379+
apiVersion: k8s.nginx.org/v1alpha1
380+
kind: TransportServer
381+
metadata:
382+
name: cafe
383+
spec:
384+
host: cafe.example.com
385+
serverSnippets: |
386+
deny 192.168.1.1;
387+
allow 192.168.1.0/24;
388+
upstreams:
389+
- name: tea
390+
service: tea-svc
391+
port: 80
392+
```
393+
394+
Snippets are intended to be used by advanced NGINX users who need more control over the generated NGINX configuration.
395+
396+
However, because of the disadvantages described below, snippets are disabled by default. To use snippets, set the [`enable-snippets`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-enable-snippets) command-line argument.
397+
398+
Disadvantages of using snippets:
399+
* *Complexity*. To use snippets, you will need to:
400+
* Understand NGINX configuration primitives and implement a correct NGINX configuration.
401+
* Understand how the IC generates NGINX configuration so that a snippet doesn't interfere with the other features in the configuration.
402+
* *Decreased robustness*. An incorrect snippet makes the NGINX config invalid which will lead to a failed reload. This will prevent any new configuration updates, including updates for the other TransportServer resource until the snippet is fixed.
403+
* *Security implications*. Snippets give access to NGINX configuration primitives and those primitives are not validated by the Ingress Controller.
404+
405+
406+
> Note that during a period when the NGINX config includes an invalid snippet, NGINX will continue to operate with the latest valid configuration.
407+
408+
373409
### Validation
374410

375411
Two types of validation are available for the TransportServer resource:

‎internal/configs/configurator.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,7 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport
559559

560560
tsCfg := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus)
561561

562-
content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(&tsCfg)
562+
content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg)
563563
if err != nil {
564564
return fmt.Errorf("Error generating TransportServer config %v: %v", name, err)
565565
}
@@ -1073,6 +1073,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, ingExes []*Ingres
10731073
return allWarnings, nil
10741074
}
10751075

1076+
// UpdateTransportServers updates TransportServers.
10761077
func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) error {
10771078
for _, tsEx := range updatedTSExes {
10781079
err := cnf.addOrUpdateTransportServer(tsEx)

‎internal/configs/transportserver.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (tsEx *TransportServerEx) String() string {
3030
}
3131

3232
// generateTransportServerConfig generates a full configuration for a TransportServer.
33-
func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool) version2.TransportServerConfig {
33+
func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool) *version2.TransportServerConfig {
3434
upstreamNamer := newUpstreamNamerForTransportServer(transportServerEx.TransportServer)
3535

3636
upstreams := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus)
@@ -59,14 +59,14 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene
5959
proxyTimeout = transportServerEx.TransportServer.Spec.SessionParameters.Timeout
6060
}
6161

62-
statusZone := ""
62+
serverSnippets := generateSnippets(true, transportServerEx.TransportServer.Spec.ServerSnippets, []string{})
63+
64+
statusZone := transportServerEx.TransportServer.Spec.Listener.Name
6365
if transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName {
6466
statusZone = transportServerEx.TransportServer.Spec.Host
65-
} else {
66-
statusZone = transportServerEx.TransportServer.Spec.Listener.Name
6767
}
6868

69-
return version2.TransportServerConfig{
69+
tsConfig := &version2.TransportServerConfig{
7070
Server: version2.StreamServer{
7171
TLSPassthrough: transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName,
7272
UnixSocket: generateUnixSocket(transportServerEx),
@@ -84,9 +84,12 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene
8484
ProxyNextUpstreamTimeout: generateTimeWithDefault(nextUpstreamTimeout, "0s"),
8585
ProxyNextUpstreamTries: nextUpstreamTries,
8686
HealthCheck: healthCheck,
87+
Snippets: serverSnippets,
8788
},
8889
Upstreams: upstreams,
8990
}
91+
92+
return tsConfig
9093
}
9194

9295
func generateUnixSocket(transportServerEx *TransportServerEx) string {

‎internal/configs/transportserver_test.go

+92-16
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package configs
22

33
import (
4-
"reflect"
54
"testing"
65

76
"github.com/google/go-cmp/cmp"
@@ -62,6 +61,82 @@ func TestTransportServerExString(t *testing.T) {
6261
}
6362
}
6463

64+
func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) {
65+
transportServerEx := TransportServerEx{
66+
TransportServer: &conf_v1alpha1.TransportServer{
67+
ObjectMeta: meta_v1.ObjectMeta{
68+
Name: "tcp-server",
69+
Namespace: "default",
70+
},
71+
Spec: conf_v1alpha1.TransportServerSpec{
72+
Listener: conf_v1alpha1.TransportServerListener{
73+
Name: "tcp-listener",
74+
Protocol: "TCP",
75+
},
76+
Upstreams: []conf_v1alpha1.Upstream{
77+
{
78+
Name: "tcp-app",
79+
Service: "tcp-app-svc",
80+
Port: 5001,
81+
},
82+
},
83+
Action: &conf_v1alpha1.Action{
84+
Pass: "tcp-app",
85+
},
86+
ServerSnippets: "deny 192.168.1.1;\nallow 192.168.1.0/24;",
87+
},
88+
},
89+
Endpoints: map[string][]string{
90+
"default/tcp-app-svc:5001": {
91+
"10.0.0.20:5001",
92+
},
93+
},
94+
}
95+
96+
listenerPort := 2020
97+
98+
expected := &version2.TransportServerConfig{
99+
Upstreams: []version2.StreamUpstream{
100+
{
101+
Name: "ts_default_tcp-server_tcp-app",
102+
Servers: []version2.StreamUpstreamServer{
103+
{
104+
Address: "10.0.0.20:5001",
105+
MaxFails: 1,
106+
FailTimeout: "10s",
107+
},
108+
},
109+
UpstreamLabels: version2.UpstreamLabels{
110+
ResourceName: "tcp-server",
111+
ResourceType: "transportserver",
112+
ResourceNamespace: "default",
113+
Service: "tcp-app-svc",
114+
},
115+
},
116+
},
117+
Server: version2.StreamServer{
118+
Port: listenerPort,
119+
UDP: false,
120+
StatusZone: "tcp-listener",
121+
ProxyPass: "ts_default_tcp-server_tcp-app",
122+
Name: "tcp-server",
123+
Namespace: "default",
124+
ProxyConnectTimeout: "60s",
125+
ProxyNextUpstream: false,
126+
ProxyNextUpstreamTries: 0,
127+
ProxyNextUpstreamTimeout: "0s",
128+
ProxyTimeout: "10m",
129+
HealthCheck: nil,
130+
Snippets: []string{"deny 192.168.1.1;", "allow 192.168.1.0/24;"},
131+
},
132+
}
133+
134+
result := generateTransportServerConfig(&transportServerEx, listenerPort, true)
135+
if diff := cmp.Diff(expected, result); diff != "" {
136+
t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff)
137+
}
138+
}
139+
65140
func TestGenerateTransportServerConfigForTCP(t *testing.T) {
66141
transportServerEx := TransportServerEx{
67142
TransportServer: &conf_v1alpha1.TransportServer{
@@ -104,7 +179,7 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) {
104179

105180
listenerPort := 2020
106181

107-
expected := version2.TransportServerConfig{
182+
expected := &version2.TransportServerConfig{
108183
Upstreams: []version2.StreamUpstream{
109184
{
110185
Name: "ts_default_tcp-server_tcp-app",
@@ -136,14 +211,15 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) {
136211
ProxyNextUpstreamTimeout: "0s",
137212
ProxyTimeout: "50s",
138213
HealthCheck: nil,
214+
Snippets: []string{},
139215
},
140216
}
141217

142-
isPlus := true
143-
result := generateTransportServerConfig(&transportServerEx, listenerPort, isPlus)
144-
if !reflect.DeepEqual(result, expected) {
145-
t.Errorf("generateTransportServerConfig() returned \n%+v but expected \n%+v", result, expected)
218+
result := generateTransportServerConfig(&transportServerEx, listenerPort, true)
219+
if diff := cmp.Diff(expected, result); diff != "" {
220+
t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff)
146221
}
222+
147223
}
148224

149225
func TestGenerateTransportServerConfigForTLSPasstrhough(t *testing.T) {
@@ -186,7 +262,7 @@ func TestGenerateTransportServerConfigForTLSPasstrhough(t *testing.T) {
186262

187263
listenerPort := 2020
188264

189-
expected := version2.TransportServerConfig{
265+
expected := &version2.TransportServerConfig{
190266
Upstreams: []version2.StreamUpstream{
191267
{
192268
Name: "ts_default_tcp-server_tcp-app",
@@ -220,13 +296,13 @@ func TestGenerateTransportServerConfigForTLSPasstrhough(t *testing.T) {
220296
ProxyNextUpstreamTries: 0,
221297
ProxyTimeout: "10m",
222298
HealthCheck: nil,
299+
Snippets: []string{},
223300
},
224301
}
225302

226-
isPlus := true
227-
result := generateTransportServerConfig(&transportServerEx, listenerPort, isPlus)
228-
if !reflect.DeepEqual(result, expected) {
229-
t.Errorf("generateTransportServerConfig() returned \n%+v but expected \n%+v", result, expected)
303+
result := generateTransportServerConfig(&transportServerEx, listenerPort, true)
304+
if diff := cmp.Diff(expected, result); diff != "" {
305+
t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff)
230306
}
231307
}
232308

@@ -275,7 +351,7 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) {
275351

276352
listenerPort := 2020
277353

278-
expected := version2.TransportServerConfig{
354+
expected := &version2.TransportServerConfig{
279355
Upstreams: []version2.StreamUpstream{
280356
{
281357
Name: "ts_default_udp-server_udp-app",
@@ -309,13 +385,13 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) {
309385
ProxyNextUpstreamTries: 0,
310386
ProxyTimeout: "10m",
311387
HealthCheck: nil,
388+
Snippets: []string{},
312389
},
313390
}
314391

315-
isPlus := true
316-
result := generateTransportServerConfig(&transportServerEx, listenerPort, isPlus)
317-
if !reflect.DeepEqual(result, expected) {
318-
t.Errorf("generateTransportServerConfig() returned \n%+v but expected \n%+v", result, expected)
392+
result := generateTransportServerConfig(&transportServerEx, listenerPort, true)
393+
if diff := cmp.Diff(expected, result); diff != "" {
394+
t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff)
319395
}
320396
}
321397

‎internal/configs/version2/nginx-plus.transportserver.tmpl

+4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ server {
2828
proxy_responses {{ $s.ProxyResponses }};
2929
{{ end }}
3030

31+
{{ range $snippet := $s.Snippets }}
32+
{{- $snippet }}
33+
{{ end }}
34+
3135
proxy_pass {{ $s.ProxyPass }};
3236

3337
{{ if $s.HealthCheck }}

‎internal/configs/version2/nginx.transportserver.tmpl

+4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ server {
2626
proxy_responses {{ $s.ProxyResponses }};
2727
{{ end }}
2828

29+
{{ range $snippet := $s.Snippets }}
30+
{{- $snippet }}
31+
{{ end }}
32+
2933
proxy_pass {{ $s.ProxyPass }};
3034

3135
proxy_timeout {{ $s.ProxyTimeout }};

‎internal/configs/version2/stream.go

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ type StreamServer struct {
3838
ProxyNextUpstreamTimeout string
3939
ProxyNextUpstreamTries int
4040
HealthCheck *StreamHealthCheck
41+
Snippets []string
4142
}
4243

4344
// StreamHealthCheck defines a health check for a StreamUpstream in a StreamServer.

‎internal/configs/virtualserver.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(vsEx *VirtualS
557557
}
558558
}
559559

560-
httpSnippets := generateSnippets(vsc.enableSnippets, vsEx.VirtualServer.Spec.HTTPSnippets, []string{""})
560+
httpSnippets := generateSnippets(vsc.enableSnippets, vsEx.VirtualServer.Spec.HTTPSnippets, []string{})
561561
serverSnippets := generateSnippets(
562562
vsc.enableSnippets,
563563
vsEx.VirtualServer.Spec.ServerSnippets,
@@ -1410,11 +1410,11 @@ func generateTimeWithDefault(value string, defaultValue string) string {
14101410
return generateTime(value)
14111411
}
14121412

1413-
func generateSnippets(enableSnippets bool, s string, defaultS []string) []string {
1414-
if !enableSnippets || s == "" {
1415-
return defaultS
1413+
func generateSnippets(enableSnippets bool, snippet string, defaultSnippets []string) []string {
1414+
if !enableSnippets || snippet == "" {
1415+
return defaultSnippets
14161416
}
1417-
return strings.Split(s, "\n")
1417+
return strings.Split(snippet, "\n")
14181418
}
14191419

14201420
func generateBuffers(s *conf_v1.UpstreamBuffers, defaultS string) string {

0 commit comments

Comments
 (0)