From 8ded9c5688fb93b3905a61504419071b73ba7409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Thu, 17 Oct 2024 10:00:27 +0800 Subject: [PATCH] add routePatch plugin (#769) Signed-off-by: spacewander --- controller/internal/istio/envoyfilter.go | 28 ++++++-- controller/internal/model/model.go | 15 ++-- .../internal/translation/merged_state.go | 42 ++++++----- controller/plugins/plugins.go | 1 + controller/plugins/routepatch/config.go | 32 +++++++++ .../plugins/testdata/http/route_patch.in.yml | 15 ++++ .../plugins/testdata/http/route_patch.out.yml | 26 +++++++ e2e/Makefile | 2 + e2e/base/nacos.yml | 2 +- e2e/pkg/suite/suite.go | 1 + e2e/tests/route_patch.go | 47 +++++++++++++ e2e/tests/route_patch.yml | 69 +++++++++++++++++++ manifests/Makefile | 4 ++ manifests/images/cp/Dockerfile | 4 +- manifests/images/dp/Dockerfile | 4 +- types/plugins/plugins.go | 1 + types/plugins/routepatch/config.go | 58 ++++++++++++++++ 17 files changed, 323 insertions(+), 28 deletions(-) create mode 100644 controller/plugins/routepatch/config.go create mode 100644 controller/plugins/testdata/http/route_patch.in.yml create mode 100644 controller/plugins/testdata/http/route_patch.out.yml create mode 100644 e2e/tests/route_patch.go create mode 100644 e2e/tests/route_patch.yml create mode 100644 types/plugins/routepatch/config.go diff --git a/controller/internal/istio/envoyfilter.go b/controller/internal/istio/envoyfilter.go index a9fd57ab..e478ae2e 100644 --- a/controller/internal/istio/envoyfilter.go +++ b/controller/internal/istio/envoyfilter.go @@ -184,6 +184,28 @@ func GenerateRouteFilter(host *model.VirtualHost, route string, config map[strin }, } + routeConfig := map[string]interface{}{} + routeFilters, _ := config[model.CategoryRouteFilter].(map[string]*fmModel.FilterConfig) + extraRouteConfig, _ := config[model.CategoryRoute].(map[string]*fmModel.FilterConfig) + if routeFilters == nil || extraRouteConfig == nil { + // bug in the code + panic("route filter and route config must be provided") + } + + if len(routeFilters) > 0 { + plainCfg := map[string]interface{}{} + for k, v := range routeFilters { + plainCfg[k] = v.Config + } + routeConfig["typed_per_filter_config"] = plainCfg + } + for _, filter := range extraRouteConfig { + fields, _ := filter.Config.(map[string]interface{}) + for k, v := range fields { + routeConfig[k] = v + } + } + return &istiov1a3.EnvoyFilter{ // We don't set ObjectMeta here because this EnvoyFilter will be merged later Spec: istioapi.EnvoyFilter{ @@ -199,9 +221,7 @@ func GenerateRouteFilter(host *model.VirtualHost, route string, config map[strin }, Patch: &istioapi.EnvoyFilter_Patch{ Operation: istioapi.EnvoyFilter_Patch_MERGE, - Value: MustNewStruct(map[string]interface{}{ - "typed_per_filter_config": config, - }), + Value: MustNewStruct(routeConfig), }, }, }, @@ -334,7 +354,7 @@ func GenerateLDSFilter(key string, ldsName string, hasHCM bool, config map[strin if cfg == nil { cfg = map[string]interface{}{} } - ecdsName := key + "-" + model.CategoryGolangPlugins + ecdsName := key + "-" + model.ECDSGolangPlugins ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches, &istioapi.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: istioapi.EnvoyFilter_HTTP_FILTER, diff --git a/controller/internal/model/model.go b/controller/internal/model/model.go index e53da5f6..c87ce7ca 100644 --- a/controller/internal/model/model.go +++ b/controller/internal/model/model.go @@ -47,9 +47,14 @@ type VirtualHost struct { } const ( - CategoryECDSGolang = "ecds_golang" - CategoryECDSListener = "ecds_listener" - CategoryECDSNetwork = "ecds_network" - CategoryListener = "listener" - CategoryGolangPlugins = "golang-filter" + CategoryECDSGolang = "ecds_golang" + CategoryECDSListener = "ecds_listener" + CategoryECDSNetwork = "ecds_network" + CategoryListener = "listener" + + CategoryRoute = "route" + CategoryRouteFilter = "route_filter" + + // This constant is used in the resource name which doesn't support '_' in the name + ECDSGolangPlugins = "golang-filter" ) diff --git a/controller/internal/translation/merged_state.go b/controller/internal/translation/merged_state.go index 2f91771f..46f3dfc7 100644 --- a/controller/internal/translation/merged_state.go +++ b/controller/internal/translation/merged_state.go @@ -92,9 +92,11 @@ const ( func translateFilterManagerConfigToPolicyInRDS(fmc *filtermanager.FilterManagerConfig, nsName *types.NamespacedName, virtualHost *model.VirtualHost) map[string]interface{} { - config := map[string]interface{}{} + nativeFilters := map[string]map[string]*fmModel.FilterConfig{ + model.CategoryRoute: {}, + model.CategoryRouteFilter: {}, + } - nativeFilters := []*fmModel.FilterConfig{} goFilterManager := &filtermanager.FilterManagerConfig{ Plugins: []*fmModel.FilterConfig{}, } @@ -143,9 +145,14 @@ func translateFilterManagerConfigToPolicyInRDS(fmc *filtermanager.FilterManagerC m = wrapper.ToRouteConfig(m) } - m["@type"] = url plugin.Config = m - nativeFilters = append(nativeFilters, plugin) + if url != "" { + filterName := fmt.Sprintf("htnn.filters.http.%s", plugin.Name) + m["@type"] = url + nativeFilters[model.CategoryRouteFilter][filterName] = plugin + } else { + nativeFilters[model.CategoryRoute][name] = plugin + } } _, ok = p.(plugins.ConsumerPlugin) @@ -173,26 +180,29 @@ func translateFilterManagerConfigToPolicyInRDS(fmc *filtermanager.FilterManagerC golangFilterName := "htnn.filters.http.golang" if ctrlcfg.EnableLDSPluginViaECDS() { - golangFilterName = virtualHost.ECDSResourceName + "-" + model.CategoryGolangPlugins + golangFilterName = virtualHost.ECDSResourceName + "-" + model.ECDSGolangPlugins } - config[golangFilterName] = map[string]interface{}{ - "@type": "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.ConfigsPerRoute", - "plugins_config": map[string]interface{}{ - "fm": map[string]interface{}{ - "config": map[string]interface{}{ - "@type": "type.googleapis.com/xds.type.v3.TypedStruct", - "value": v, + golangFilterPlugin := &fmModel.FilterConfig{ + Config: map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.ConfigsPerRoute", + "plugins_config": map[string]interface{}{ + "fm": map[string]interface{}{ + "config": map[string]interface{}{ + "@type": "type.googleapis.com/xds.type.v3.TypedStruct", + "value": v, + }, }, }, }, } + nativeFilters[model.CategoryRouteFilter][golangFilterName] = golangFilterPlugin } - for _, filter := range nativeFilters { - name := fmt.Sprintf("htnn.filters.http.%s", filter.Name) - config[name] = filter.Config + // satisfy the requirement of the returned type + config := map[string]interface{}{} + for k, v := range nativeFilters { + config[k] = v } - return config } diff --git a/controller/plugins/plugins.go b/controller/plugins/plugins.go index 4e5c4d35..e80d1b30 100644 --- a/controller/plugins/plugins.go +++ b/controller/plugins/plugins.go @@ -24,6 +24,7 @@ import ( _ "mosn.io/htnn/controller/plugins/localratelimit" _ "mosn.io/htnn/controller/plugins/lua" _ "mosn.io/htnn/controller/plugins/networkrbac" + _ "mosn.io/htnn/controller/plugins/routepatch" _ "mosn.io/htnn/controller/plugins/tlsinspector" _ "mosn.io/htnn/types/plugins" ) diff --git a/controller/plugins/routepatch/config.go b/controller/plugins/routepatch/config.go new file mode 100644 index 00000000..8c655a3f --- /dev/null +++ b/controller/plugins/routepatch/config.go @@ -0,0 +1,32 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routepatch + +import ( + "mosn.io/htnn/api/pkg/plugins" + "mosn.io/htnn/types/plugins/routepatch" +) + +func init() { + plugins.RegisterPlugin(routepatch.Name, &plugin{}) +} + +type plugin struct { + routepatch.Plugin +} + +func (p *plugin) ConfigTypeURL() string { + return "" +} diff --git a/controller/plugins/testdata/http/route_patch.in.yml b/controller/plugins/testdata/http/route_patch.in.yml new file mode 100644 index 00000000..05d3c831 --- /dev/null +++ b/controller/plugins/testdata/http/route_patch.in.yml @@ -0,0 +1,15 @@ +apiVersion: htnn.mosn.io/v1 +kind: FilterPolicy +metadata: + name: policy + namespace: default +spec: + targetRef: + group: networking.istio.io + kind: VirtualService + name: default + filters: + routePatch: + config: + route: + cluster_header: "Cluster-Name" diff --git a/controller/plugins/testdata/http/route_patch.out.yml b/controller/plugins/testdata/http/route_patch.out.yml new file mode 100644 index 00000000..eae2aa2f --- /dev/null +++ b/controller/plugins/testdata/http/route_patch.out.yml @@ -0,0 +1,26 @@ +- metadata: + creationTimestamp: null + name: htnn-h-default.local + namespace: default + spec: + configPatches: + - applyTo: HTTP_ROUTE + match: + routeConfiguration: + vhost: + name: default.local:80 + route: + name: default/default + patch: + operation: MERGE + value: + route: + cluster_header: Cluster-Name + status: {} +- metadata: + creationTimestamp: null + name: htnn-http-filter + namespace: istio-system + spec: + priority: -10 + status: {} diff --git a/e2e/Makefile b/e2e/Makefile index 898fc5ab..8cc358c6 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -45,12 +45,14 @@ delete-cluster: kind .PHONY: e2e-prepare-controller-image e2e-prepare-controller-image: kind cd ../manifests/ && CONTROLLER_IMAGE=htnn/controller:e2e CONTROLLER_BASE_IMAGE=$(DOCKER_MIRROR)docker.io/istio/pilot:$(ISTIO_VERSION) \ + GO_BUILD_BASE_IMAGE=$(DOCKER_MIRROR)docker.io/golang:1.21 \ make build-controller-image $(KIND) load docker-image -n htnn htnn/controller:e2e .PHONY: e2e-prepare-data-plane-image e2e-prepare-data-plane-image: kind cd ../manifests/ && PROXY_BASE_IMAGE=$(DOCKER_MIRROR)docker.io/istio/proxyv2:$(ISTIO_VERSION) \ + GO_BUILD_BASE_IMAGE=$(DOCKER_MIRROR)docker.io/golang:1.21 \ PROXY_IMAGE=htnn/gateway:e2e make build-proxy-image $(KIND) load docker-image htnn/gateway:e2e --name htnn diff --git a/e2e/base/nacos.yml b/e2e/base/nacos.yml index f70c4d86..7134296a 100644 --- a/e2e/base/nacos.yml +++ b/e2e/base/nacos.yml @@ -29,7 +29,7 @@ spec: spec: containers: - name: nacos - image: nacos/nacos-server:v1.4.6-slim + image: m.daocloud.io/docker.io/nacos/nacos-server:v1.4.6-slim ports: - containerPort: 8848 env: diff --git a/e2e/pkg/suite/suite.go b/e2e/pkg/suite/suite.go index d4dadf58..4385de2f 100644 --- a/e2e/pkg/suite/suite.go +++ b/e2e/pkg/suite/suite.go @@ -136,6 +136,7 @@ func (suite *Suite) waitDeployments(t *testing.T) { } { cmdline := fmt.Sprintf("kubectl wait --timeout=5m -n %s deployment/%s --for=condition=Available", cond.ns, cond.name) + t.Logf("start waiting for deployment %s in namespace %s, cmd: %s", cond.name, cond.ns, cmdline) cmd := strings.Fields(cmdline) wait := exec.Command(cmd[0], cmd[1:]...) err := wait.Run() diff --git a/e2e/tests/route_patch.go b/e2e/tests/route_patch.go new file mode 100644 index 00000000..95a55d2b --- /dev/null +++ b/e2e/tests/route_patch.go @@ -0,0 +1,47 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "mosn.io/htnn/e2e/pkg/suite" +) + +func init() { + suite.Register(suite.Test{ + Run: func(t *testing.T, suite *suite.Suite) { + tr := &http.Transport{DialContext: func(ctx context.Context, proto, addr string) (conn net.Conn, err error) { + return net.DialTimeout("tcp", ":18000", 1*time.Second) + }} + client := &http.Client{Transport: tr, Timeout: 10 * time.Second} + rsp, err := client.Get("http://default.local:18000/echo") + require.NoError(t, err) + require.Equal(t, 403, rsp.StatusCode) + rsp, err = client.Get("http://default.local:18000/echo2") + require.NoError(t, err) + require.Equal(t, 403, rsp.StatusCode) + rsp, err = client.Get("http://default.local:18000/") + require.NoError(t, err) + require.Equal(t, 405, rsp.StatusCode) + }, + }) +} diff --git a/e2e/tests/route_patch.yml b/e2e/tests/route_patch.yml new file mode 100644 index 00000000..d25eb749 --- /dev/null +++ b/e2e/tests/route_patch.yml @@ -0,0 +1,69 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: vs + namespace: istio-system +spec: + gateways: + - default + hosts: + - "default.local" + http: + - match: + - uri: + prefix: /echo + route: + - destination: + host: backend + port: + number: 8080 + - match: + - uri: + prefix: /echo2 + route: + - destination: + host: backend + port: + number: 8080 + - match: + - uri: + prefix: / + name: last + route: + - destination: + host: backend + port: + number: 8080 +--- +apiVersion: htnn.mosn.io/v1 +kind: FilterPolicy +metadata: + name: policy + namespace: istio-system +spec: + targetRef: + group: networking.istio.io + kind: VirtualService + name: vs + filters: + routePatch: + config: + directResponse: + status: 403 +--- +apiVersion: htnn.mosn.io/v1 +kind: FilterPolicy +metadata: + name: policy2 + namespace: istio-system +spec: + targetRef: + group: networking.istio.io + kind: VirtualService + name: vs + sectionName: last + filters: + routePatch: + config: + directResponse: + status: 405 diff --git a/manifests/Makefile b/manifests/Makefile index d7fcc197..760d2519 100644 --- a/manifests/Makefile +++ b/manifests/Makefile @@ -24,10 +24,12 @@ PROXY_IMAGE ?= htnn/proxy:latest PROXY_BASE_IMAGE ?= istio/proxyv2:$(ISTIO_VERSION) CONTROLLER_IMAGE ?= htnn/controller:latest CONTROLLER_BASE_IMAGE ?= docker.io/istio/pilot:$(ISTIO_VERSION) +GO_BUILD_BASE_IMAGE ?= golang:1.21 .PHONY: build-proxy-image build-proxy-image: cd .. && $(CONTAINER_TOOL) build -t ${PROXY_IMAGE} --build-arg GOPROXY=${GOPROXY} --build-arg PROXY_BASE_IMAGE=${PROXY_BASE_IMAGE} \ + --build-arg GO_BUILD_BASE_IMAGE=${GO_BUILD_BASE_IMAGE} \ -f manifests/images/dp/Dockerfile . # If you wish to build the controller image targeting other platforms you can use the --platform flag. @@ -37,6 +39,7 @@ build-proxy-image: build-controller-image: cd .. && $(CONTAINER_TOOL) build -t ${CONTROLLER_IMAGE} \ --build-arg GOPROXY=${GOPROXY} --build-arg CONTROLLER_BASE_IMAGE=${CONTROLLER_BASE_IMAGE} \ + --build-arg GO_BUILD_BASE_IMAGE=${GO_BUILD_BASE_IMAGE} \ -f manifests/images/cp/Dockerfile . # PLATFORMS defines the target platforms for the image be built to provide support to multiple @@ -54,6 +57,7 @@ docker-buildx: ## Build and push docker image for cross-platform support $(CONTAINER_TOOL) buildx use project-v3-builder && \ $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${CONTROLLER_IMAGE} \ --build-arg GOPROXY=${GOPROXY} --build-arg CONTROLLER_BASE_IMAGE=${CONTROLLER_BASE_IMAGE} \ + --build-arg GO_BUILD_BASE_IMAGE=${GO_BUILD_BASE_IMAGE} \ -f /tmp/Dockerfile.cross . ; \ $(CONTAINER_TOOL) buildx rm project-v3-builder rm /tmp/Dockerfile.cross diff --git a/manifests/images/cp/Dockerfile b/manifests/images/cp/Dockerfile index 7e9f635a..0fdc49eb 100644 --- a/manifests/images/cp/Dockerfile +++ b/manifests/images/cp/Dockerfile @@ -14,8 +14,10 @@ # Dockerfile has specific requirement to put this ARG at the beginning: # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG GO_BUILD_BASE_IMAGE ARG CONTROLLER_BASE_IMAGE -FROM golang:1.21 as builder +# hadolint ignore=DL3006 +FROM ${GO_BUILD_BASE_IMAGE} as builder ARG TARGETOS ARG TARGETARCH ARG GOPROXY diff --git a/manifests/images/dp/Dockerfile b/manifests/images/dp/Dockerfile index eb2a9886..7717a902 100644 --- a/manifests/images/dp/Dockerfile +++ b/manifests/images/dp/Dockerfile @@ -14,8 +14,10 @@ # Dockerfile has specific requirement to put this ARG at the beginning: # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG GO_BUILD_BASE_IMAGE ARG PROXY_BASE_IMAGE -FROM golang:1.21 as builder +# hadolint ignore=DL3006 +FROM ${GO_BUILD_BASE_IMAGE} as builder ARG TARGETOS ARG TARGETARCH ARG GOPROXY diff --git a/types/plugins/plugins.go b/types/plugins/plugins.go index 318dda4a..169d0fd6 100644 --- a/types/plugins/plugins.go +++ b/types/plugins/plugins.go @@ -37,5 +37,6 @@ import ( _ "mosn.io/htnn/types/plugins/networkrbac" _ "mosn.io/htnn/types/plugins/oidc" _ "mosn.io/htnn/types/plugins/opa" + _ "mosn.io/htnn/types/plugins/routepatch" _ "mosn.io/htnn/types/plugins/tlsinspector" ) diff --git a/types/plugins/routepatch/config.go b/types/plugins/routepatch/config.go new file mode 100644 index 00000000..48f0ce8b --- /dev/null +++ b/types/plugins/routepatch/config.go @@ -0,0 +1,58 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routepatch + +import ( + "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" + + "mosn.io/htnn/api/pkg/filtermanager/api" + "mosn.io/htnn/api/pkg/plugins" +) + +const ( + Name = "routePatch" +) + +func init() { + plugins.RegisterPluginType(Name, &Plugin{}) +} + +type Plugin struct { + plugins.PluginMethodDefaultImpl +} + +func (p *Plugin) Order() plugins.PluginOrder { + return plugins.PluginOrder{ + Position: plugins.OrderPositionInner, + } +} + +func (p *Plugin) Type() plugins.PluginType { + return plugins.TypeGeneral +} + +func (p *Plugin) Config() api.PluginConfig { + return &CustomConfig{} +} + +type CustomConfig struct { + route.Route +} + +func (conf *CustomConfig) Validate() error { + // We can't use the default validation because the route is not a complete route. + // Skip the validation for now. + return nil +}