diff --git a/cmd/otelcorecol/builder-config.yaml b/cmd/otelcorecol/builder-config.yaml index 827ee97b80d..6f04d0568d8 100644 --- a/cmd/otelcorecol/builder-config.yaml +++ b/cmd/otelcorecol/builder-config.yaml @@ -83,6 +83,7 @@ replaces: - go.opentelemetry.io/collector/extension/extensioncapabilities => ../../extension/extensioncapabilities - go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware - go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + - go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter - go.opentelemetry.io/collector/extension/extensiontest => ../../extension/extensiontest - go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension - go.opentelemetry.io/collector/extension/xextension => ../../extension/xextension diff --git a/cmd/otelcorecol/go.mod b/cmd/otelcorecol/go.mod index 6b59f19e448..194ed3c1181 100644 --- a/cmd/otelcorecol/go.mod +++ b/cmd/otelcorecol/go.mod @@ -4,7 +4,7 @@ module go.opentelemetry.io/collector/cmd/otelcorecol go 1.23.0 -toolchain go1.23.10 +toolchain go1.24.1 require ( go.opentelemetry.io/collector/component v1.34.0 @@ -111,7 +111,9 @@ require ( go.opentelemetry.io/collector/exporter/xexporter v0.128.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensioncapabilities v0.128.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/extension/extensiontest v0.128.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect @@ -266,6 +268,8 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../ext replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/extension/extensiontest => ../../extension/extensiontest replace go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension diff --git a/component/component.go b/component/component.go index 5a32c5041d7..0daa19254c9 100644 --- a/component/component.go +++ b/component/component.go @@ -77,6 +77,20 @@ func (f ShutdownFunc) Shutdown(ctx context.Context) error { return f(ctx) } +type componentImpl struct { + StartFunc + ShutdownFunc +} + +var _ Component = componentImpl{} + +func NewComponentImpl(sf StartFunc, shf ShutdownFunc) Component { + return componentImpl{ + StartFunc: sf, + ShutdownFunc: shf, + } +} + // Kind represents component kinds. type Kind struct { name string @@ -110,6 +124,10 @@ const ( StabilityLevelStable ) +func (sl *StabilityLevel) Self() StabilityLevel { + return *sl +} + func (sl *StabilityLevel) UnmarshalText(in []byte) error { str := strings.ToLower(string(in)) switch str { @@ -194,3 +212,17 @@ type CreateDefaultConfigFunc func() Config func (f CreateDefaultConfigFunc) CreateDefaultConfig() Config { return f() } + +type factoryImpl struct { + TypeFunc + CreateDefaultConfigFunc +} + +var _ Factory = factoryImpl{} + +func NewFactoryImpl(tf TypeFunc, cf CreateDefaultConfigFunc) Factory { + return factoryImpl{ + TypeFunc: tf, + CreateDefaultConfigFunc: cf, + } +} diff --git a/component/identifiable.go b/component/identifiable.go index 6b814768161..11948f20a4f 100644 --- a/component/identifiable.go +++ b/component/identifiable.go @@ -38,6 +38,11 @@ func (t Type) String() string { return t.name } +// Self returns itself. +func (t Type) Self() Type { + return t +} + // MarshalText marshals returns the Type name. func (t Type) MarshalText() ([]byte, error) { return []byte(t.name), nil @@ -71,6 +76,16 @@ func MustNewType(strType string) Type { return ty } +// TypeFunc is ... +type TypeFunc func() Type + +// Type gets the type of the component created by this factory. +func (f TypeFunc) Type() Type { + if f == nil { + } + return f() +} + // ID represents the identity for a component. It combines two values: // * type - the Type of the component. // * name - the name of that component. diff --git a/config/configgrpc/client_middleware_test.go b/config/configgrpc/client_middleware_test.go index 4bd3dcf8cfc..0fac0184b50 100644 --- a/config/configgrpc/client_middleware_test.go +++ b/config/configgrpc/client_middleware_test.go @@ -31,9 +31,7 @@ type testClientMiddleware struct { } func newTestMiddlewareConfig(name string) configmiddleware.Config { - return configmiddleware.Config{ - ID: component.MustNewID(name), - } + return configmiddleware.Config(component.MustNewID(name)) } func newTestClientMiddleware(name string) extension.Extension { @@ -162,9 +160,7 @@ func TestClientMiddlewareToClientErrors(t *testing.T) { Insecure: true, }, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -182,9 +178,7 @@ func TestClientMiddlewareToClientErrors(t *testing.T) { Insecure: true, }, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "get options failed", diff --git a/config/configgrpc/configgrpc.go b/config/configgrpc/configgrpc.go index c55cbe2a87e..33a9ca37a0b 100644 --- a/config/configgrpc/configgrpc.go +++ b/config/configgrpc/configgrpc.go @@ -207,7 +207,7 @@ type ServerConfig struct { IncludeMetadata bool `mapstructure:"include_metadata,omitempty"` // Middlewares for the gRPC server. - Middlewares []configmiddleware.Config `mapstructure:"middlewares,omitempty"` + Middlewares []configmiddleware.Config `mapstructure:"middleware,omitempty"` // prevent unkeyed literal initialization _ struct{} diff --git a/config/configgrpc/go.mod b/config/configgrpc/go.mod index 93abdb6efa4..a02cced744c 100644 --- a/config/configgrpc/go.mod +++ b/config/configgrpc/go.mod @@ -17,7 +17,7 @@ require ( go.opentelemetry.io/collector/extension v1.34.0 go.opentelemetry.io/collector/extension/extensionauth v1.34.0 go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.128.0 - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 go.opentelemetry.io/collector/pdata v1.34.0 go.opentelemetry.io/collector/pdata/testdata v0.128.0 @@ -44,6 +44,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect @@ -59,6 +60,7 @@ require ( golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect + golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -105,3 +107,7 @@ replace go.opentelemetry.io/collector/pipeline => ../../pipeline replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer diff --git a/config/configgrpc/go.sum b/config/configgrpc/go.sum index cef34798683..f785b3d2c61 100644 --- a/config/configgrpc/go.sum +++ b/config/configgrpc/go.sum @@ -107,6 +107,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/config/configgrpc/server_middleware_test.go b/config/configgrpc/server_middleware_test.go index 0a0b736dff8..598a2dff8ff 100644 --- a/config/configgrpc/server_middleware_test.go +++ b/config/configgrpc/server_middleware_test.go @@ -122,9 +122,7 @@ func TestServerMiddlewareToServerErrors(t *testing.T) { Transport: confignet.TransportTypeTCP, }, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -142,9 +140,7 @@ func TestServerMiddlewareToServerErrors(t *testing.T) { Transport: confignet.TransportTypeTCP, }, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "get server options failed", diff --git a/config/confighttp/client_middleware_test.go b/config/confighttp/client_middleware_test.go index 01155962613..c8c96f18bec 100644 --- a/config/confighttp/client_middleware_test.go +++ b/config/confighttp/client_middleware_test.go @@ -61,9 +61,7 @@ func newTestClientMiddleware(name string) component.Component { } func newTestClientConfig(name string) configmiddleware.Config { - return configmiddleware.Config{ - ID: component.MustNewID(name), - } + return configmiddleware.Config(component.MustNewID(name)) } func TestClientMiddlewares(t *testing.T) { @@ -162,9 +160,7 @@ func TestClientMiddlewareErrors(t *testing.T) { config: ClientConfig{ Endpoint: server.URL, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -179,9 +175,7 @@ func TestClientMiddlewareErrors(t *testing.T) { config: ClientConfig{ Endpoint: server.URL, Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "http middleware error", @@ -216,9 +210,7 @@ func TestGRPCClientMiddlewareErrors(t *testing.T) { config: ClientConfig{ Endpoint: "localhost:1234", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -233,9 +225,7 @@ func TestGRPCClientMiddlewareErrors(t *testing.T) { config: ClientConfig{ Endpoint: "localhost:1234", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "grpc middleware error", diff --git a/config/confighttp/go.mod b/config/confighttp/go.mod index 0b053d82a42..e14c79227de 100644 --- a/config/confighttp/go.mod +++ b/config/confighttp/go.mod @@ -19,7 +19,7 @@ require ( go.opentelemetry.io/collector/extension v1.34.0 go.opentelemetry.io/collector/extension/extensionauth v1.34.0 go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.128.0 - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 go.opentelemetry.io/collector/featuregate v1.34.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 @@ -42,6 +42,7 @@ require ( github.com/hashicorp/go-version v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect go.opentelemetry.io/collector/pdata v1.34.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect @@ -72,6 +73,8 @@ replace go.opentelemetry.io/collector/config/configtls => ../configtls replace go.opentelemetry.io/collector/extension => ../../extension +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/extension/extensionauth => ../../extension/extensionauth replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware @@ -95,3 +98,7 @@ replace go.opentelemetry.io/collector/pipeline => ../../pipeline replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile diff --git a/config/confighttp/server_middleware_test.go b/config/confighttp/server_middleware_test.go index ca383ccfe38..5a559890f35 100644 --- a/config/confighttp/server_middleware_test.go +++ b/config/confighttp/server_middleware_test.go @@ -47,9 +47,7 @@ func newTestServerMiddleware(name string) component.Component { } func newTestServerConfig(name string) configmiddleware.Config { - return configmiddleware.Config{ - ID: component.MustNewID(name), - } + return configmiddleware.Config(component.MustNewID(name)) } func TestServerMiddleware(t *testing.T) { @@ -154,9 +152,7 @@ func TestServerMiddlewareErrors(t *testing.T) { config: ServerConfig{ Endpoint: "localhost:0", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -171,9 +167,7 @@ func TestServerMiddlewareErrors(t *testing.T) { config: ServerConfig{ Endpoint: "localhost:0", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "http middleware error", @@ -209,9 +203,7 @@ func TestServerMiddlewareErrors(t *testing.T) { config: ServerConfig{ Endpoint: "localhost:0", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("nonexistent"), - }, + configmiddleware.Config(component.MustNewID("nonexistent")), }, }, errText: "failed to resolve middleware \"nonexistent\": middleware not found", @@ -226,9 +218,7 @@ func TestServerMiddlewareErrors(t *testing.T) { config: ServerConfig{ Endpoint: "localhost:0", Middlewares: []configmiddleware.Config{ - { - ID: component.MustNewID("errormw"), - }, + configmiddleware.Config(component.MustNewID("errormw")), }, }, errText: "grpc middleware error", diff --git a/config/confighttp/xconfighttp/go.mod b/config/confighttp/xconfighttp/go.mod index 34b9d34cde4..d71f1972401 100644 --- a/config/confighttp/xconfighttp/go.mod +++ b/config/confighttp/xconfighttp/go.mod @@ -23,7 +23,10 @@ require ( github.com/google/go-tpm v0.9.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/cors v1.11.1 // indirect @@ -35,11 +38,17 @@ require ( go.opentelemetry.io/collector/config/configmiddleware v0.128.0 // indirect go.opentelemetry.io/collector/config/configopaque v1.34.0 // indirect go.opentelemetry.io/collector/config/configtls v1.34.0 // indirect + go.opentelemetry.io/collector/consumer v1.34.0 // indirect + go.opentelemetry.io/collector/consumer/xconsumer v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect go.opentelemetry.io/collector/pdata v1.34.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/log v0.12.2 // indirect @@ -94,3 +103,9 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../../ replace go.opentelemetry.io/collector/config/configmiddleware => ../../configmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../../pdata/pprofile diff --git a/config/confighttp/xconfighttp/go.sum b/config/confighttp/xconfighttp/go.sum index d70395fee0f..bff6b8f165d 100644 --- a/config/confighttp/xconfighttp/go.sum +++ b/config/confighttp/xconfighttp/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -25,6 +26,7 @@ github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/go-tpm-tools v0.4.4 h1:oiQfAIkc6xTy9Fl5NKTeTJkBTlXdHsxAofmQyxBKY98= github.com/google/go-tpm-tools v0.4.4/go.mod h1:T8jXkp2s+eltnCDIsXR84/MTcVU9Ja7bh3Mit0pa4AY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= @@ -39,6 +41,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -51,6 +54,8 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/config/configmiddleware/configmiddleware.go b/config/configmiddleware/configmiddleware.go index 831ba7766fe..1ac2899a198 100644 --- a/config/configmiddleware/configmiddleware.go +++ b/config/configmiddleware/configmiddleware.go @@ -11,9 +11,13 @@ import ( "fmt" "net/http" + "go.uber.org/multierr" "google.golang.org/grpc" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension/extensionlimiter" + grpclimiter "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper/grpc" + httplimiter "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper/http" "go.opentelemetry.io/collector/extension/extensionmiddleware" ) @@ -26,11 +30,17 @@ var ( ) // Middleware defines the extension ID for a middleware component. -type Config struct { - // ID specifies the name of the extension to use. - ID component.ID `mapstructure:"id,omitempty"` - // prevent unkeyed literal initialization - _ struct{} +type Config component.ID + +func (cfg *Config) UnmarshalText(in []byte) error { + var id component.ID + err := id.UnmarshalText(in) + *cfg = Config(id) + return err +} + +func resolveFailed(id component.ID) error { + return fmt.Errorf("failed to resolve middleware %q: %w", id, errMiddlewareNotFound) } // GetHTTPClientRoundTripper attempts to select the appropriate @@ -39,13 +49,20 @@ type Config struct { // found, an error is returned. This should only be used by HTTP // clients. func (m Config) GetHTTPClientRoundTripper(_ context.Context, extensions map[component.ID]component.Component) (func(http.RoundTripper) (http.RoundTripper, error), error) { - if ext, found := extensions[m.ID]; found { + if ext, found := extensions[component.ID(m)]; found { if client, ok := ext.(extensionmiddleware.HTTPClient); ok { return client.GetHTTPRoundTripper, nil } + if limiter, ok := ext.(extensionlimiter.AnyProvider); ok { + limiter, err := httplimiter.NewClientLimiter(limiter) + if err != nil { + return nil, err + } + return limiter.GetHTTPRoundTripper, nil + } return nil, errNotHTTPClient } - return nil, fmt.Errorf("failed to resolve middleware %q: %w", m.ID, errMiddlewareNotFound) + return nil, resolveFailed(component.ID(m)) } // GetHTTPServerHandler attempts to select the appropriate @@ -54,14 +71,21 @@ func (m Config) GetHTTPClientRoundTripper(_ context.Context, extensions map[comp // found, an error is returned. This should only be used by HTTP // servers. func (m Config) GetHTTPServerHandler(_ context.Context, extensions map[component.ID]component.Component) (func(http.Handler) (http.Handler, error), error) { - if ext, found := extensions[m.ID]; found { + if ext, found := extensions[component.ID(m)]; found { if server, ok := ext.(extensionmiddleware.HTTPServer); ok { return server.GetHTTPHandler, nil } + if limiter, ok := ext.(extensionlimiter.AnyProvider); ok { + limiter, err := httplimiter.NewServerLimiter(limiter) + if err != nil { + return nil, err + } + return limiter.GetHTTPHandler, nil + } return nil, errNotHTTPServer } - return nil, fmt.Errorf("failed to resolve middleware %q: %w", m.ID, errMiddlewareNotFound) + return nil, resolveFailed(component.ID(m)) } // GetGRPCClientOptions attempts to select the appropriate @@ -69,13 +93,20 @@ func (m Config) GetHTTPServerHandler(_ context.Context, extensions map[component // returns the gRPC dial options. If a middleware is not found, an // error is returned. This should only be used by gRPC clients. func (m Config) GetGRPCClientOptions(_ context.Context, extensions map[component.ID]component.Component) ([]grpc.DialOption, error) { - if ext, found := extensions[m.ID]; found { + if ext, found := extensions[component.ID(m)]; found { if client, ok := ext.(extensionmiddleware.GRPCClient); ok { return client.GetGRPCClientOptions() } + if limiter, ok := ext.(extensionlimiter.AnyProvider); ok { + lim, err := grpclimiter.NewClientLimiter(limiter) + if err != nil { + return nil, err + } + return lim.GetGRPCClientOptions() + } return nil, errNotGRPCClient } - return nil, fmt.Errorf("failed to resolve middleware %q: %w", m.ID, errMiddlewareNotFound) + return nil, resolveFailed(component.ID(m)) } // GetGRPCServerOptions attempts to select the appropriate @@ -83,12 +114,42 @@ func (m Config) GetGRPCClientOptions(_ context.Context, extensions map[component // returns the gRPC server options. If a middleware is not found, an // error is returned. This should only be used by gRPC servers. func (m Config) GetGRPCServerOptions(_ context.Context, extensions map[component.ID]component.Component) ([]grpc.ServerOption, error) { - if ext, found := extensions[m.ID]; found { + if ext, found := extensions[component.ID(m)]; found { if server, ok := ext.(extensionmiddleware.GRPCServer); ok { return server.GetGRPCServerOptions() } + if limiter, ok := ext.(extensionlimiter.AnyProvider); ok { + lim, err := grpclimiter.NewServerLimiter(limiter) + if err != nil { + return nil, err + } + return lim.GetGRPCServerOptions() + } return nil, errNotGRPCServer } - return nil, fmt.Errorf("failed to resolve middleware %q: %w", m.ID, errMiddlewareNotFound) + return nil, resolveFailed(component.ID(m)) +} + +// GetBaseLimiters gets a list of basic limiters. These can be +// upgraded to any kind of limiter, subject to restrictions documented +// in that extension interface, using limiterhelper.MiddlewaresToLimiterProvider. +func GetBaseLimiters(host component.Host, cfgs []Config) ([]extensionlimiter.AnyProvider, error) { + var err error + var lims []extensionlimiter.AnyProvider + all := host.GetExtensions() + for _, m := range cfgs { + ext, ok := all[component.ID(m)] + if !ok { + err = multierr.Append(err, resolveFailed(component.ID(m))) + continue + } + if lim, ok := ext.(extensionlimiter.AnyProvider); ok { + lims = append(lims, lim) + } else { + // Note: In this case, we skip the middleware + // that is not a limiter. + } + } + return lims, err } diff --git a/config/configmiddleware/go.mod b/config/configmiddleware/go.mod index afec289462a..4a420a51865 100644 --- a/config/configmiddleware/go.mod +++ b/config/configmiddleware/go.mod @@ -6,8 +6,10 @@ require ( github.com/stretchr/testify v1.10.0 go.opentelemetry.io/collector/component v1.34.0 go.opentelemetry.io/collector/extension v1.34.0 - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 + go.uber.org/multierr v1.11.0 google.golang.org/grpc v1.73.0 ) @@ -18,22 +20,28 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/consumer v1.34.0 // indirect + go.opentelemetry.io/collector/consumer/xconsumer v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect go.opentelemetry.io/collector/pdata v1.34.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/log v0.12.2 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/net v0.39.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.24.0 // indirect + golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -54,3 +62,11 @@ replace go.opentelemetry.io/collector/pipeline => ../../pipeline replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/extension => ../../extension + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/consumer => ../../consumer diff --git a/config/configmiddleware/go.sum b/config/configmiddleware/go.sum index f5ab1e98e7b..58543183a81 100644 --- a/config/configmiddleware/go.sum +++ b/config/configmiddleware/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -11,20 +12,30 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -76,6 +87,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/docs/rfcs/functional-composition-pattern.md b/docs/rfcs/functional-composition-pattern.md new file mode 100644 index 00000000000..1d2a40244ef --- /dev/null +++ b/docs/rfcs/functional-composition-pattern.md @@ -0,0 +1,300 @@ +# Functional Composition Pattern + +## Overview + +This codebase uses a **Functional Composition** pattern for its core +interfaces. This pattern decomposes individual methods from single- +and multi-method interface types into individual function types, then +recomposes them into a concrete implementations, which may be sealed +(or not), and exported (or not) depending on the context. + +This approach provides flexibility, testability, and safe interface +evolution. + +## Core Principles + +### 1. Decompose Interfaces into Function Types + +Instead of implementing interfaces directly on structs, we create function types for each method: + +```go +// Interface definition +type RateReservation interface { + WaitTime() time.Duration + Cancel() +} + +// Function types for each method +type WaitTimeFunc func() time.Duration +type CancelFunc func() + +// Function types implement their corresponding methods +func (f WaitTimeFunc) WaitTime() time.Duration { + if f == nil { + return 0 + } + return f() +} + +func (f CancelFunc) Cancel() { + if f == nil { + return + } + f() +} +``` + +Users of the `net/http` package will have seen this pattern +before. The `http.HandlerFunc` type can be seen as a prototype for +functional composition of HTTP handlers. Interestingly, the +single-method `http.RoundTripper` interface, which represents the same +interaction on the client-side, does not have a `RoundTripperFunc`. In +this codebase, the pattern is applied exstensively. + +Note that each function type always implements no-op functionality, +such that passing `nil` corresponds with the no-op implementation for +a that function. + +### 2. Compose Functions into Interface Implementations + +Create concrete implementations embed the corresponding function type: + +```go +type rateReservationImpl struct { + WaitTimeFunc + CancelFunc +} + +// Constructor for an instance +func NewRateReservationImpl(wf WaitTimeFunc, cf CancelFunc) RateReservation { + return rateReservationImpl{ + WaitTimeFunc: wf, + CancelFunc: cf, + } +} +``` + +Note that because the argument to NewXyzImpl() is a list of +`<...>Func` objects, and the Go compiler automatically converts func +values, so we can write: + +```go + return NewRateReservationImpl( + // Wait time 1 second + func() time.Duration { return time.Second }, + // Cancel is a no-op. + nil, + ) +``` + +For constant values and enumerated types, define a `Self()` method to +act as the corresponding function implementation: + +```go +// Self returns itself. +func (t Type) Self() Type { + return t +} + +// TypeFunc is ... +type TypeFunc func() Type + +// Type gets the type of the component created by this factory. +func (f TypeFunc) Type() Type { + if f == nil { + } + return f() +} +``` + +For example, we can decompose, modify, and recompose a +`component.Factory`: + +```go + // Construct a factory with a new default Config: + factory := sometype.NewFactory() + cfg := factory.CreateDefaultConfig() + // modify the config object + return NewFactoryImpl(factory.Type().Self, cfg.Self) +``` + +### 3. Use Constructors for Interface Values + +Provide constructor functions rather than exposing concrete types: + +```go +// Good: Constructor function +func NewRateLimiterImpl(f ReserveRateFunc) RateLimiter { + return rateLimiterImpl{ReserveRateFunc: f} +} + +// Avoid: Direct struct instantiation +// rateLimiterImpl{ReserveRateFunc: f} // Don't do this +``` + +Rarely should the implementation type be exposed. This pattern can be +combined with the Functional Option pattern (from `receiver/receiver.go`): + +```go +// Setup optional logging-related functions +func WithLogs(createLogs CreateLogsFunc, sl component.StabilityLevel) FactoryOption { + return factoryOptionFunc(func(o *factoryImpl, cfgType component.Type) { + o.CreateLogsFunc = createLogs + o.LogsStabilityFunc = sl.Self + }) +} + +// Accept options to configure various aspects of the interface +func NewFactory(cfgType component.Type, createDefaultConfig component.CreateDefaultConfigFunc, options ...FactoryOption) Factory { + f := factoryImpl{ + Factory: component.NewFactoryImpl(cfgType.Self, createDefaultConfig), + } + for _, opt := range options { + opt.applyOption(&f, cfgType) + } + return f +} +``` + +## Rationale + +### Flexibility and Composition + +This pattern enables composition scenarios by making it easy to +compose and decompose interface values. + +```go +// Transform existing factories with cross-cutting concerns +func NewLimitedFactory(fact receiver.Factory, cfgf LimiterConfigurator) receiver.Factory { + return receiver.NewFactoryImpl( + fact.Type, + fact.CreateDefaultConfig, + receiver.CreateTracesFunc(limitReceiver(fact.CreateTraces, traceTraits{}, cfgf)), + fact.TracesStability, + receiver.CreateMetricsFunc(limitReceiver(fact.CreateMetrics, metricTraits{}, cfgf)), + fact.MetricsStability, + receiver.CreateLogsFunc(limitReceiver(fact.CreateLogs, logTraits{}, cfgf)), + fact.LogsStability, + ) +} +``` + +This is sometimes called aspect-oriented programming, for example to +add logging to an existing function: + +```go +func addLogging(f ReserveRateFunc) ReserveRateFunc { + return func(ctx context.Context, n int) (RateReservation, error) { + log.Printf("Reserving rate for %d", n) + return f(ctx, n) + } +} +``` + +### Safe Interface Evolution + +Using a private method allows sealing the interface type, which forces +external users to use functional constructor methods. This allows +interfaces to evolve safely because users are forced to use +constructor functions. + +```go +type RateLimiter interface { + ReserveRate(context.Context, int) (RateReservation, error) + + // Can add new methods without breaking existing code + private() // Prevents external implementations +} +``` + +### Enhanced Testability + +Individual methods can be tested independently: + +```go +func TestWaitTimeFunction(t *testing.T) { + waitFn := WaitTimeFunc(func() time.Duration { return time.Second }) + assert.Equal(t, time.Second, waitFn.WaitTime()) +} +``` + +## Implementation Guidelines + +### 1. Interface Design + +- Define interfaces with clear method signatures +- Include a `private()` method if you need to control implementations +- Keep interfaces focused and cohesive + +### 2. Function Type Naming + +- Use `Func` naming convention +- Ensure function signatures match the interface method exactly +- Always implement the corresponding interface method on the function type + +### 3. Nil Handling + +Always handle nil function types gracefully to act as a no-op implementation. + +```go +func (f ReserveRateFunc) ReserveRate(ctx context.Context, value int) (RateReservation, error) { + if f == nil { + return NewRateReservationImpl(nil, nil), nil // No-op implementation + } + return f(ctx, value) +} +``` + +### 4. Constructor Patterns + +Follow consistent constructor naming for building an implementation +of each interface: + +```go +// For interfaces: NewImpl +func NewRateLimiterImpl(f ReserveRateFunc) RateLimiter +``` + +Follow consistent type naming for each method-function type: + +```go +// For function types: Func +type ReserveRateFunc func(context.Context, int) (RateReservation, error) +``` + +### 5. Implementation Structs + +- Use unexported names for implementation structs +- Embed function types directly +- Implement private methods for sealing the interface + +```go +type RateLimiter interface { + // ... + + // Must use functional constructors outside this package + private() +} + +type rateLimiterImpl struct { + ReserveRateFunc +} + +func (rateLimiterImpl) private() {} +``` + +## When to Use This Pattern + +### Appropriate Use Cases + +- **Interfaces with single or multiple methods** that benefit from composition +- **Cross-cutting concerns** that need to be applied selectively +- **Interface evolution** where you need to add methods over time +- **Factory patterns** where you're assembling behavior from components +- **Middleware/decorator scenarios** where you're transforming behavior + +### When to Avoid + +- **Stateful objects** where methods need to share significant state +- **Performance-critical code** where function call overhead matters +- **Simple implementations** where the pattern adds unnecessary complexity diff --git a/exporter/otlpexporter/go.mod b/exporter/otlpexporter/go.mod index 2fe5b0fca07..f732f700e8d 100644 --- a/exporter/otlpexporter/go.mod +++ b/exporter/otlpexporter/go.mod @@ -65,7 +65,9 @@ require ( go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 // indirect go.opentelemetry.io/collector/extension v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect @@ -88,6 +90,7 @@ require ( golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect + golang.org/x/time v0.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) @@ -132,8 +135,6 @@ replace go.opentelemetry.io/collector/client => ../../client replace go.opentelemetry.io/collector/config/configretry => ../../config/configretry -replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest replace go.opentelemetry.io/collector/receiver/xreceiver => ../../receiver/xreceiver @@ -177,4 +178,8 @@ replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/co replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../../pdata/xpdata diff --git a/exporter/otlpexporter/go.sum b/exporter/otlpexporter/go.sum index 18835a6ddf1..884425faf8b 100644 --- a/exporter/otlpexporter/go.sum +++ b/exporter/otlpexporter/go.sum @@ -124,6 +124,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/exporter/otlphttpexporter/go.mod b/exporter/otlphttpexporter/go.mod index a0736a96017..7b6792b3d6b 100644 --- a/exporter/otlphttpexporter/go.mod +++ b/exporter/otlphttpexporter/go.mod @@ -65,7 +65,9 @@ require ( go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 // indirect go.opentelemetry.io/collector/extension v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect @@ -130,8 +132,6 @@ replace go.opentelemetry.io/collector/consumer => ../../consumer replace go.opentelemetry.io/collector/config/configretry => ../../config/configretry -replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest replace go.opentelemetry.io/collector/client => ../../client @@ -175,4 +175,8 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../ext replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../../pdata/xpdata diff --git a/extension/extensionlimiter/Makefile b/extension/extensionlimiter/Makefile new file mode 100644 index 00000000000..ded7a36092d --- /dev/null +++ b/extension/extensionlimiter/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/extension/extensionlimiter/README.md b/extension/extensionlimiter/README.md new file mode 100644 index 00000000000..6bac9ca6f0e --- /dev/null +++ b/extension/extensionlimiter/README.md @@ -0,0 +1,333 @@ +# OpenTelemetry Collector Extension Limiter Package + +**Document status: development** + +The `extensionlimiter` package provides interfaces for limiting +pipelines in the OpenTelemetry Collector, enabling control over data +flow and resource usage through extensions which are configured +through middleware and/or directly by pipeline components. + +## Overview + +This package defines two foundational limiter **kinds**, with similar +but distinct interfaces. A limiter extension can be: + +- **Rate Limiter**: Controls time-based limits over weights such as + bytes or items per second. +- **Resource Limiter**: Controls physical limits over weights such as + concurrent requests or active memory in use. + +Requests are quantified with an integer value and identified by +**weight key**, indicating the type of quantity being measured and +limited. There are currently four weight keys with a standard +definition: + +1. Network bytes (compressed) +2. Request count +3. Request items +4. Request bytes (uncompressed) + +## Early-as-possible application + +Limiter extensions should be used as early as possible in a +pipeline. There are two automatic ways that receivers can integrate +with rate limiters: + +- Middleware application: rate limiters are automatically recognized + in the list of middleware. Middleware supports HTTP and gRPC, client + and server, unary and streaming cases. Middleware automatically + implements request bytes and request count limits. +- Consumer application: rate limiters can be applied before the next + consumer in the pipeline, using standard `consumerlimiter.LimiterConfig` + configuration. + +Limiters should be applied, if possible, before work on a request +begins. Work that is done before a limit is requested is subject to +loss, in case the limiter causes failure. + +Limiters are not always applied in receivers, but all receivers should +support limiters through middleware and/or `consumerlimiter`. + +## Delay-the-caller application + +Limiters should be applied so that they delay the caller. This is an +important case of the early-as-possible rule: limit requests should be +made before returning control, in order to slow the process that is +contributing to the limit. + +In order to support delaying the caller in complex scenarios, +non-blocking interfaces are provided for each of the limiter +interfaces. Non-blocking APIs allow callers to delay the caller while +requesting the limit and then to perform their work asynchronously. + +The `limiterhelper.Wrapper` limiter interface is provided which +simplifies the application of limits to a scoped callback, making it +easy to use a blocking limit request. + +## Failure options + +Limiters at their discretion can block or fail requests that would +exceed a limit. The decision may influenced by limiter configuration +(e.g., burst, maximum wait parameters) and/or the deadline of the +request context. When the delay is small, it is usually beneficial to +wait instead of failing because (a) avoids the wasted effort (e.g., +re-transmitting data), (b) delays the caller for the effect of +back-pressure. + +When a limiter returns failure, the client should return a +protocol-specific failure code indicating resource exhaustion. The +recognized resource exhaustion codes are HTTP 429 and gRPC +RESOURCE_EXHAUSTED. Receivers should follow protocol-specific +recommendations, which for +[OTLP](https://opentelemetry.io/docs/specs/otlp/) includes returning a +`RetryDelay` parameter. + +If the limit request results in waiting, limiters should delay to +allow the request to proceed, however they should give up and return +at some point. As a recommendation, receivers and other components +should allow requests to wait up to configurable fraction of their +deadline. If the request cannot enter a pipeline before for example +half of its deadline, return failure instead of allowing it to +proceed. + +### Built-in limiters + +#### MemoryLimiter extension + +The `memorylimiterextension` gives access to an internal component +named `MemoryLimiter` with an interface named `MustDeny()`. +Components can call this component directly, however when configured +as a limiter extension, this component is modeled as a `RateLimiter` +that is on or off based on the current result of `MustDeny`. + +#### RateLimiter + +A built-in helper implementation of the RateLimiter interface is +provided, based on `golang.org/x/time/rate.Limter`. These underlying +rate limiters are parameterized by two numbers: + +- `limit` (float64): the maximum frequency of weight-units per second +- `burst` (uint64): the "burst" value of the Token-bucket algorithm. + +#### ResourceLimiter + +A built-in helper implementation of the ResourceLimiter interface is +provided, based on a bounded queue with LIFO behavior. These +underlying resource limiters are parameterized by two numbers: + +- `request` (uint64): the maximum of concurrent resource value admitted +- `waiting` (uint64): the maximum of concurrent resource value permitted to wait + +### Examples + +#### OTLP receiver + +Limiters applied through middleware and/or via receiver-level +limiters. Middleware limiters are automatically configured using +`configgrpc` or `confighttp`. Receivers can add support at the factory +level using helpers in `consumerlimiter`. + +For the OTLP receiver (e.g., with three `ratelimiter` extensions and a +`resourcelimiter` extension): + +```yaml +extensions: + ratelimiter/limit_for_grpc: + network_bytes: ... + + ratelimiter/limit_for_http: + network_bytes: ... + + ratelimiter/limit_items: + request_items: ... + + resourcelimiter/limit_memory: + request_bytes: ... + +receivers: + otlp: + protocols: + grpc: + middleware: + - ratelimiter/limit_for_grpc + - resourcelimiter/limit_memory + http: + middleware: + - ratelimiter/limit_for_http + - resourcelimiter/limit_memory + limiters: + request_items: ratelimiter/limit_items +``` + +Note that in general, middleware components do not have access to the +number of items in a request, so users are directed to receiver-level +`limiters` configuration to limit items. + +#### HTTP metrics scraper + +A HTTP pull-based receiver can implement a basic limited scraper loop +as follows. The HTTP client config object's `middlewares` field +automatically configures network bytes and request count limits: + +```yaml +receivers: + httpscraper: + http: + middleware: + - ratelimiter/scraper_request_bytes + - compression/zstd + - ratelimiter/scraper_network_bytes + limiters: + request_count: ratelimiter/scraper_count + request_items: ratelimiter/scraper_items +``` + +##### Data-dependent limits + +When a single unit of data contains limits that are assignable to +multiple distinct limiters, we can use the non-blocking rate limiter +interface and drop data that would exceed a limit. For example, to +limit based on metadata extracted from the OpenTelemetry resource +value: + +``` +func (p *processor) limitLogs(ctx context.Context, logsData plog.Logs) (plog.Logs, extensionlimiter.ReleaseFunc, error) { + var rels extensionlimiter.ReleaseFuncs + logsData.ResourceLogs().RemoveIf(func(rl plog.ResourceLogs) bool { + // For an individual resource, ... + md := resourceToMetadata(rl.Resource()) + reservation, err := p.limiter.ReserveRate(withMetadata(ctx, md)) + if err != nil { + return false + } + if reservation.WaitTime() > 0 { + reservation.Cancel() + return false + } + default: + return true + } + }) + if logsData.ResourceLogs().Len() == 0 { + return logsData, func() {}, processorhelper.ErrSkipProcessingData + } + return logsData, rels.Release, nil +} + +func (p *processor) ConsumeLogs(ctx context.Context, logsData plog.Logs) error { + logsData, release, err = limitLogs(ctx, logsData) + if err != nil { + return err + } + defer release() + return p.nextLogs.ConsumeLogs(ctx, logsData) +} +``` + +Here, the limiter's `ReserveRate` function does not block the caller, +allowing the processor to drop data instead. Note the call to +`RateReservation.Cancel` undoes the effect of the untaken reservation. +The same approach works for `ResourceLimiter` as well using using +`ResourceReservation`, its `Delay` channel `Release` function. + +#### Open questions + +##### Provider options + +An `Option` type has been added as a placeholder in the provider +interfaces. **NOTE: No options are implemented.** Potential options: + +- The protocol name +- The signal kind +- The caller's component ID + +Because the set of each of these is small, it is possible to +pre-compute limiter instances for the cross product of configurations. + +##### Middleware and/or LimiterConfig + +The question is how we avoid double-count certain limits whether they +are implemented in middleware, through a factory, through custom +receiver code, or other. + +In the current limiter extension proposal, middleware references are +component IDs (without referce to weight key), while `LimiterConfig` +is a set of weight-specific limiter references. Users will be able to +double-count certain features when they user the same limiter +extension in both middleware and limiters configuration, unless we +help explicitly avoid this scnario. It could be accomplished using +context variables or start-time settings. + +##### Controlling middleware order + +While middleware order is a list of components, it is difficult for +users to reason about the order of application. To implement a +network-bytes or request-bytes limit, the limiter has to be configured +before or after the compression middleware. + +Today, HTTP middleware for compression is automatically inserted +although [it could become middleware +itself](https://github.com/open-telemetry/opentelemetry-collector/issues/13228). + +Since middlware references are simple identifiers (without weight +keys), additional help is needed to distinguish compressed bytes from +uncompressed bytes, especially for the HTTP cases. Potentially, +middleware can look at Transfer-Encoding or context.Context values +to distinguish these cases. + +#### Middleware component syntax + +In many discussions and documents, a number of authors have shown a +preference for simple component identifiers, without the leading `id:` +field syntax, which is to change + +```go +type Config struct { + ID component.ID `mapstructure:"id,omitempty"` +} +``` + +to: + +```go +type Config struct component.ID +``` + +This allows middleware to be listed as simple identifiers, + +```yaml +receivers: + otlp: + protocols: + grpc: + middleware: + - ratelimiter/1 + - ratelimiter/2 + limiters: + request_bytes: admissionlimiter/3 +``` + +##### Are built-in rate and resource limiters needed? + +The provided helper implementations are based in +`golang.org/x/time/rate` and +`collector-contrib/internal/otelarrow/admission2`. We could instead +create two extension implementations for these. The code is a hundred +lines or so each. + +The [Elastic rate limiter +processor](https://github.com/elastic/opentelemetry-collector-components/blob/main/processor/ratelimitprocessor/README.md) +would be a good contribution for the community. We are interested in +real-world features such as the ability to set `metadata_keys`. + +##### Instrumentation + +It is possible to extract Counter (RateLimiter) and UpDownCounter +(ResourceLimiter) instrumentation to convey the rates and totals +associated with each limiter. + +This should investigated along with investigation into the topics +raised above. Users will be well served if we are able to confidently +extract network-bytes, request-bytes, request-items, and request-count +information without double-counting and provide instrumentation +covering these variables. diff --git a/extension/extensionlimiter/any_limiter.go b/extension/extensionlimiter/any_limiter.go new file mode 100644 index 00000000000..fdbef709395 --- /dev/null +++ b/extension/extensionlimiter/any_limiter.go @@ -0,0 +1,19 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// AnyProvider is an any limiter implementation, possibly one of the +// limiter interfaces. This serves as a marker for implementations +// which provider rate limiters of any (one)kind. +type AnyProvider interface { + // unexportedProviderFunc may be embedded using an AnyProviderImpl. + unexportedProviderFunc() +} + +// AnyProviderImpl can be embedded as a marker that a component +// implements one of the rate limiters. +type AnyProviderImpl struct {} + +func (AnyProviderImpl) unexportedProviderFunc() {} + diff --git a/extension/extensionlimiter/go.mod b/extension/extensionlimiter/go.mod new file mode 100644 index 00000000000..025c9e2e53a --- /dev/null +++ b/extension/extensionlimiter/go.mod @@ -0,0 +1,74 @@ +module go.opentelemetry.io/collector/extension/extensionlimiter + +go 1.23.0 + +require ( + go.opentelemetry.io/collector/component v1.34.0 + go.opentelemetry.io/collector/consumer v1.34.0 + go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.126.0 + go.opentelemetry.io/collector/pdata v1.34.0 + go.opentelemetry.io/collector/pdata/pprofile v0.128.0 + go.opentelemetry.io/collector/receiver v1.34.0 + go.opentelemetry.io/collector/receiver/xreceiver v0.0.0-00010101000000-000000000000 + go.uber.org/multierr v1.11.0 + golang.org/x/time v0.11.0 + google.golang.org/grpc v1.73.0 +) + +require ( + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/extension v1.34.0 // indirect + go.opentelemetry.io/collector/featuregate v1.34.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect + go.opentelemetry.io/collector/pipeline v0.128.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect + go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel/log v0.12.2 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) + +replace go.opentelemetry.io/collector/consumer => ../../consumer + +replace go.opentelemetry.io/collector/featuregate => ../../featuregate + +replace go.opentelemetry.io/collector/component => ../../component + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/pdata => ../../pdata + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware + +replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/extension => ../ + +replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry + +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware + +replace go.opentelemetry.io/collector/receiver => ../../receiver + +replace go.opentelemetry.io/collector/receiver/xreceiver => ../../receiver/xreceiver diff --git a/extension/extensionlimiter/go.sum b/extension/extensionlimiter/go.sum new file mode 100644 index 00000000000..4b826c4b806 --- /dev/null +++ b/extension/extensionlimiter/go.sum @@ -0,0 +1,103 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/collector/consumer/consumertest v0.128.0 h1:x50GB0I/QvU3sQuNCap5z/P2cnq2yHoRJ/8awkiT87w= +go.opentelemetry.io/collector/consumer/consumertest v0.128.0/go.mod h1:Wb3IAbMY/DOIwJPy81PuBiW2GnKoNIz4THE7wfJwovE= +go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 h1:u2E32P7j1a/gRgZDWhIXC+Shd4rLg70mnE7QLI/Ssnw= +go.opentelemetry.io/contrib/bridges/otelzap v0.11.0/go.mod h1:pJPCLM8gzX4ASqLlyAXjHBEYxgbOQJ/9bidWxD6PEPQ= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/log v0.12.2 h1:yob9JVHn2ZY24byZeaXpTVoPS6l+UrrxmxmPKohXTwc= +go.opentelemetry.io/otel/log v0.12.2/go.mod h1:ShIItIxSYxufUMt+1H5a2wbckGli3/iCfuEbVZi/98E= +go.opentelemetry.io/otel/log/logtest v0.0.0-20250526142609-aa5bd0e64989 h1:4JF7oY9CcHrPGfBLijDcXZyCzGckVEyOjuat5ktmQRg= +go.opentelemetry.io/otel/log/logtest v0.0.0-20250526142609-aa5bd0e64989/go.mod h1:NToOxLDCS1tXDSB2dIj44H9xGPOpKr0csIN+gnuihv4= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/extension/extensionlimiter/limiterhelper/consumerlimiter/consumerlimiter.go b/extension/extensionlimiter/limiterhelper/consumerlimiter/consumerlimiter.go new file mode 100644 index 00000000000..5b2b078914b --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/consumerlimiter/consumerlimiter.go @@ -0,0 +1,343 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package consumerlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper/consumerlimiter" + +import ( + "context" + "errors" + "fmt" + + "go.uber.org/multierr" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/xconsumer" + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/pprofile" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/xreceiver" +) + +var ( + ErrLimiterNotFound = errors.New("limiter not found") + ErrNotALimiter = errors.New("not a limiter") + ErrLimiterUnsupported = errors.New("limiter unsupported") +) + +// Config is the standard pipeline configuration for limiting a +// consumer interface by specific signal. +// +// This type should be embedded, for example: +// +// // LimiterConfig allows applying limiter extensions for request count, items, and bytes. +// consumerlimiter.LimiterConfig `mapstructure:"limiters"` +// +type LimiterConfig struct { + RequestCount component.ID `mapstructure:"request_count"` + RequestItems component.ID `mapstructure:"request_items"` + RequestBytes component.ID `mapstructure:"request_bytes"` +} + +// capable is an internal interface describing common features of a +// consumer. +type capable interface { + Capabilities() consumer.Capabilities +} + +// Traits object interface is generalized by P the pipeline data type +// (e.g., ptrace.Traces) and C the consumer type (e.g., +// consumer.Traces) and R the return component type. +type traits[P, C, R any] interface { + // itemCount is SpanCount(), DataPointCount(), or LogRecordCount(). + itemCount(P) int + // requestBytes uses the appropriate protobuf Bytesr as a proxy + // for memory used. + requestSize(data P) int + // consume calls the appropriate consumer method (e.g., ConsumeTraces) + consume(ctx context.Context, data P, next C) error + // create is a functional constructor the consumer type (e.g., consumer.NewTraces) + create(func(ctx context.Context, data P) error, ...consumer.Option) (C, error) + // newReceiver constructs the correct receiver type + newReceiver(component.Component) R +} + +// Creator is the function to create a receiver components. +type creator[C capable, R component.Component] func(ctx context.Context, set receiver.Settings, cfg component.Config, next C) (R, error) + +// Traces traits + +type traceTraits struct{} + +var _ traits[ptrace.Traces, consumer.Traces, receiver.Traces] = traceTraits{} + +func (traceTraits) itemCount(data ptrace.Traces) int { + return data.SpanCount() +} + +func (traceTraits) requestSize(data ptrace.Traces) int { + var sizer ptrace.MarshalSizer + return sizer.TracesSize(data) +} + +func (traceTraits) create(next func(ctx context.Context, data ptrace.Traces) error, opts ...consumer.Option) (consumer.Traces, error) { + return consumer.NewTraces(next, opts...) +} + +func (traceTraits) consume(ctx context.Context, data ptrace.Traces, next consumer.Traces) error { + return next.ConsumeTraces(ctx, data) +} + +func (traceTraits) newReceiver(c component.Component) receiver.Traces { + return receiver.Traces(c) +} + +// Metrics traits + +type metricTraits struct{} + +var _ traits[pmetric.Metrics, consumer.Metrics, receiver.Metrics] = metricTraits{} + +func (metricTraits) itemCount(data pmetric.Metrics) int { + return data.DataPointCount() +} + +func (metricTraits) requestSize(data pmetric.Metrics) int { + var sizer pmetric.MarshalSizer + return sizer.MetricsSize(data) +} + +func (metricTraits) create(next func(ctx context.Context, data pmetric.Metrics) error, opts ...consumer.Option) (consumer.Metrics, error) { + return consumer.NewMetrics(next, opts...) +} + +func (metricTraits) consume(ctx context.Context, data pmetric.Metrics, next consumer.Metrics) error { + return next.ConsumeMetrics(ctx, data) +} + +func (metricTraits) newReceiver(c component.Component) receiver.Metrics { + return receiver.Metrics(c) +} + +// Logs traits + +type logTraits struct{} + +var _ traits[plog.Logs, consumer.Logs, receiver.Logs] = logTraits{} + +func (logTraits) itemCount(data plog.Logs) int { + return data.LogRecordCount() +} + +func (logTraits) requestSize(data plog.Logs) int { + var sizer plog.MarshalSizer + return sizer.LogsSize(data) +} + +func (logTraits) create(next func(ctx context.Context, data plog.Logs) error, opts ...consumer.Option) (consumer.Logs, error) { + return consumer.NewLogs(next, opts...) +} + +func (logTraits) consume(ctx context.Context, data plog.Logs, next consumer.Logs) error { + return next.ConsumeLogs(ctx, data) +} + +func (logTraits) newReceiver(c component.Component) receiver.Logs { + return receiver.Logs(c) +} + +// Profiles traits + +type profileTraits struct{} + +var _ traits[pprofile.Profiles, xconsumer.Profiles, xreceiver.Profiles] = profileTraits{} + +func (profileTraits) itemCount(data pprofile.Profiles) int { + return data.SampleCount() +} + +func (profileTraits) requestSize(data pprofile.Profiles) int { + var sizer pprofile.MarshalSizer + return sizer.ProfilesSize(data) +} + +func (profileTraits) create(next func(ctx context.Context, data pprofile.Profiles) error, opts ...consumer.Option) (xconsumer.Profiles, error) { + return xconsumer.NewProfiles(next, opts...) +} + +func (profileTraits) consume(ctx context.Context, data pprofile.Profiles, next xconsumer.Profiles) error { + return next.ConsumeProfiles(ctx, data) +} + +func (profileTraits) newReceiver(c component.Component) xreceiver.Profiles { + return xreceiver.Profiles(c) +} + +type limitedReceiver[P any, C capable, R component.Component, T traits[P, C, R]] struct { + cfg LimiterConfig + next C + self T + component.ShutdownFunc +} + +func (l *limitedReceiver[P, C, R, T]) Capabilities() consumer.Capabilities { + return l.next.Capabilities() +} + +func (l *limitedReceiver[P, C, R, T]) Start(ctx context.Context, host component.Host) error { + var unset component.ID + var err1, err2, err3 error + if name := l.cfg.RequestBytes; name != unset { + l.next, err1 = l.limitOne( + host, + name, + extensionlimiter.WeightKeyRequestBytes, + func(data P) int { + return l.self.requestSize(data) + }, + ) + } + if name := l.cfg.RequestItems; name != unset { + l.next, err2 = l.limitOne( + host, + name, + extensionlimiter.WeightKeyRequestItems, + func(data P) int { + return l.self.itemCount(data) + }, + ) + } + if name := l.cfg.RequestCount; name != unset { + l.next, err3 = l.limitOne( + host, + name, + extensionlimiter.WeightKeyRequestCount, + func(data P) int { + return 1 + }, + ) + } + + return multierr.Append(err1, multierr.Append(err2, err3)) +} + +func (l *limitedReceiver[P, C, R, T]) consume(ctx context.Context, data P) error { + return l.self.consume(ctx, data, l.next) +} + +// limitOne obtains a Wrapper and applies a single weight limit. +func (l *limitedReceiver[P, C, R, T]) limitOne( + host component.Host, + name component.ID, + key extensionlimiter.WeightKey, + quantify func(P) int, +) (C, error) { + exts := host.GetExtensions() + comp := exts[name] + if comp == nil { + return l.next, fmt.Errorf("%w: %s", ErrLimiterNotFound, name.String()) + } + alim, isLim := comp.(extensionlimiter.AnyProvider) + if !isLim { + return l.next, fmt.Errorf("%w: %s", ErrNotALimiter, name.String()) + } + provider, err := limiterhelper.AnyToWrapperProvider(alim) + if err != nil { + return l.next, err + } + // Note: not passing options to GetWrapper(), an open question. + lim, err := provider.GetWrapper(key) + if err != nil { + return l.next, err + } + if lim == nil { + return l.next, nil + } + return l.self.create(func(ctx context.Context, data P) error { + return lim.LimitCall(ctx, quantify(data), func(ctx context.Context) error { + return l.self.consume(ctx, data, l.next) + }) + }, consumer.WithCapabilities(l.next.Capabilities())) +} + +// newLimited is signal-generic limiting logic. +func newLimited[P any, C capable, R component.Component]( + next C, + cfg LimiterConfig, + self traits[P, C, R], +) *limitedReceiver[P, C, R, traits[P, C, R]] { + return &limitedReceiver[P, C, R, traits[P, C, R]]{ + cfg: cfg, + next: next, + self: self, + } +} + +// LimiterConfigurator lets components configure limiters using +// a field they determine. +type LimiterConfigurator func(component.Config) LimiterConfig + +// limitReceiver limits a receiver component where P is pipeline data, +// C is the consumer type, and R is the return type. +func limitReceiver[P any, C capable, R component.Component]( + cf creator[C, R], + t traits[P, C, R], + cfgf LimiterConfigurator, +) creator[C, R] { + return func(ctx context.Context, set receiver.Settings, cfg component.Config, next C) (R, error) { + var limiter *limitedReceiver[P, C, R, traits[P, C, R]] + var emptyCfg LimiterConfig + if lc := cfgf(cfg); lc != emptyCfg { + limiter = newLimited(next, lc, t) + var err error + next, err = t.create(limiter.consume) + if err != nil { + var zero R + return zero, err + } + } + + recv, err := cf(ctx, set, cfg, next) + if err != nil { + return recv, err + } + if limiter == nil { + return recv, nil + } + return t.newReceiver(component.NewComponentImpl( + func (ctx context.Context, host component.Host) error { + err1 := limiter.Start(ctx, host) + err2 := recv.Start(ctx, host) + return multierr.Append(err1, err2) + }, + func (ctx context.Context) error { + err1 := recv.Shutdown(ctx) + err2 := limiter.Shutdown(ctx) + return multierr.Append(err1, err2) + }, + )), nil + } +} + +func NewLimitedFactory(fact xreceiver.Factory, cfgf LimiterConfigurator) xreceiver.Factory { + return xreceiver.NewFactoryImpl( + receiver.NewFactoryImpl( + component.NewFactoryImpl( + fact.Type, + fact.CreateDefaultConfig, + ), + receiver.CreateTracesFunc(limitReceiver(fact.CreateTraces, traceTraits{}, cfgf)), + fact.TracesStability, + receiver.CreateMetricsFunc(limitReceiver(fact.CreateMetrics, metricTraits{}, cfgf)), + fact.MetricsStability, + receiver.CreateLogsFunc(limitReceiver(fact.CreateLogs, logTraits{}, cfgf)), + fact.LogsStability, + ), + xreceiver.CreateProfilesFunc(limitReceiver(fact.CreateProfiles, profileTraits{}, cfgf)), + fact.ProfilesStability, + ) +} diff --git a/extension/extensionlimiter/limiterhelper/grpc/grpclimiter.go b/extension/extensionlimiter/limiterhelper/grpc/grpclimiter.go new file mode 100644 index 00000000000..2e6f7c9c11e --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/grpc/grpclimiter.go @@ -0,0 +1,299 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package grpclimiter + +import ( + "context" + "sync" + "time" + + "go.uber.org/multierr" + "google.golang.org/grpc" + "google.golang.org/grpc/stats" + + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + "go.opentelemetry.io/collector/extension/extensionmiddleware" +) + +// contextKey is a private type for context keys +type contextKey int + +const ( + rateLimiterErrorKey contextKey = iota +) + +// rateLimiterErrorState holds error state,allowing rate limiters to return +// errors in the correct context. +type rateLimiterErrorState struct { + mu sync.Mutex + err error +} + +// checkRateLimiterError checks if there's a prior rate limiter error in the context. +func checkRateLimiterError(ctx context.Context) error { + if state, ok := ctx.Value(rateLimiterErrorKey).(*rateLimiterErrorState); ok { + state.mu.Lock() + defer state.mu.Unlock() + return state.err + } + return nil +} + +// setRateLimiterError sets a rate limiter error in the context +func setRateLimiterError(ctx context.Context, err error) { + if state, ok := ctx.Value(rateLimiterErrorKey).(*rateLimiterErrorState); ok { + state.mu.Lock() + defer state.mu.Unlock() + state.err = multierr.Append(state.err, err) + } +} + +func NewClientLimiter(ext extensionlimiter.AnyProvider) (extensionmiddleware.GRPCClient, error) { + wp, err1 := limiterhelper.AnyToWrapperProvider(ext) + rp, err2 := limiterhelper.AnyToRateLimiterProvider(ext) + if err := multierr.Append(err1, err2); err != nil { + return nil, err + } + requestLimiter, err3 := wp.GetWrapper(extensionlimiter.WeightKeyRequestCount) + compressedLimiter, err4 := rp.GetRateLimiter(extensionlimiter.WeightKeyNetworkBytes) + uncompressedLimiter, err5 := rp.GetRateLimiter(extensionlimiter.WeightKeyRequestBytes) + + if err := multierr.Append(err3, multierr.Append(err4, err5)); err != nil { + return nil, err + } + + var gopts []grpc.DialOption + if requestLimiter != nil { + gopts = append(gopts, grpc.WithUnaryInterceptor( + func( + ctxIn context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + return requestLimiter.LimitCall( + ctxIn, 1, + func(ctx context.Context) error { + if err := checkRateLimiterError(ctx); err != nil { + return err + } + return invoker(ctx, method, req, reply, cc, opts...) + }) + }), + grpc.WithStreamInterceptor( + func( + ctxIn context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + cstream, err := streamer(ctxIn, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return wrapClientStream(cstream, method, requestLimiter), nil + }), + ) + } + if compressedLimiter != nil || uncompressedLimiter != nil { + gopts = append(gopts, grpc.WithStatsHandler( + &limiterStatsHandler{ + compressedLimiter: compressedLimiter, + uncompressedLimiter: uncompressedLimiter, + isClient: true, + })) + } + return extensionmiddleware.GetGRPCClientOptionsFunc(func() ([]grpc.DialOption, error) { + return gopts, nil + }), nil +} + +func NewServerLimiter(ext extensionlimiter.AnyProvider) (extensionmiddleware.GRPCServer, error) { + wp, err1 := limiterhelper.AnyToWrapperProvider(ext) + rp, err2 := limiterhelper.AnyToRateLimiterProvider(ext) + if err := multierr.Append(err1, err2); err != nil { + return nil, err + } + requestLimiter, err3 := wp.GetWrapper(extensionlimiter.WeightKeyRequestCount) + compressedLimiter, err4 := rp.GetRateLimiter(extensionlimiter.WeightKeyNetworkBytes) + uncompressedLimiter, err5 := rp.GetRateLimiter(extensionlimiter.WeightKeyRequestBytes) + if err := multierr.Append(err3, multierr.Append(err4, err5)); err != nil { + return nil, err + } + + var gopts []grpc.ServerOption + if requestLimiter != nil { + gopts = append(gopts, grpc.ChainUnaryInterceptor( + func( + ctxIn context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (any, error) { + var resp any + err := requestLimiter.LimitCall( + ctxIn, 1, + func(ctx context.Context) error { + if err := checkRateLimiterError(ctx); err != nil { + return err + } + var err error + resp, err = handler(ctx, req) + return err + }) + return resp, err + }), grpc.ChainStreamInterceptor( + func( + srv interface{}, + ss grpc.ServerStream, + info *grpc.StreamServerInfo, + handler grpc.StreamHandler, + ) error { + return handler(srv, wrapServerStream(ss, info, requestLimiter)) + }), + ) + } + if compressedLimiter != nil || uncompressedLimiter != nil { + gopts = append(gopts, grpc.StatsHandler( + &limiterStatsHandler{ + compressedLimiter: compressedLimiter, + uncompressedLimiter: uncompressedLimiter, + isClient: false, + })) + } + + return extensionmiddleware.GetGRPCServerOptionsFunc(func() ([]grpc.ServerOption, error) { + return gopts, nil + }), nil +} + +// limiterStatsHandler implements the stats.Handler interface for rate limiting. +type limiterStatsHandler struct { + compressedLimiter extensionlimiter.RateLimiter + uncompressedLimiter extensionlimiter.RateLimiter + isClient bool +} + +func (h *limiterStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} + +func (h *limiterStatsHandler) HandleRPC(ctx context.Context, s stats.RPCStats) { + // Check for payload messages to apply network byte rate limiting + var wireBytes int + var reqBytes int + switch payload := s.(type) { + case *stats.InPayload: + // Server receiving payload (or client receiving response) + if !h.isClient { + wireBytes = payload.WireLength + reqBytes = payload.Length + } + case *stats.OutPayload: + // Client sending payload (or server sending response) + if h.isClient { + wireBytes = payload.WireLength + reqBytes = payload.Length + } + default: + // Not a payload message, no rate limiting to apply + return + } + + // Implement 1 or 2 rate limits in parallel + var err1, err2 error + var res1, res2 extensionlimiter.RateReservation + + if wireBytes != 0 { + res1, err1 = h.compressedLimiter.ReserveRate(ctx, wireBytes) + } + if reqBytes != 0 { + res2, err2 = h.uncompressedLimiter.ReserveRate(ctx, reqBytes) + } + + var wait1, wait2 time.Duration + + if res1 != nil { + wait1 = res1.WaitTime() + defer res1.Cancel() + } + if res2 != nil { + wait2 = res2.WaitTime() + defer res2.Cancel() + } + + if err := multierr.Append(err1, err2); err != nil { + setRateLimiterError(ctx, err) + return + } + + wait := max(wait1, wait2) + select { + case <-ctx.Done(): + case <-time.After(wait): + } +} + +func (h *limiterStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + // Create a new context with rate limiter error state + return context.WithValue(ctx, rateLimiterErrorKey, &rateLimiterErrorState{}) +} + +func (h *limiterStatsHandler) HandleConn(ctx context.Context, _ stats.ConnStats) { +} + +type serverStream struct { + grpc.ServerStream + limiter limiterhelper.Wrapper +} + +// RecvMsg applies rate limiting to server stream message receiving. +func (s *serverStream) RecvMsg(m any) error { + return s.limiter.LimitCall( + s.Context(), 1, + func(ctx context.Context) error { + if err := checkRateLimiterError(ctx); err != nil { + return err + } + return s.ServerStream.RecvMsg(m) + }) +} + +// wrapServerStream wraps a gRPC server stream with rate limiting. +func wrapServerStream(ss grpc.ServerStream, _ *grpc.StreamServerInfo, limiter limiterhelper.Wrapper) grpc.ServerStream { + return &serverStream{ + ServerStream: ss, + limiter: limiter, + } +} + +type clientStream struct { + grpc.ClientStream + limiter limiterhelper.Wrapper +} + +// SendMsg applies rate limiting to client stream message sending. +func (s *clientStream) SendMsg(m any) error { + return s.limiter.LimitCall( + s.Context(), 1, + func(ctx context.Context) error { + if err := checkRateLimiterError(ctx); err != nil { + return err + } + return s.ClientStream.SendMsg(m) + }) +} + +// wrapClientStream wraps a gRPC client stream with rate limiting. +func wrapClientStream(cs grpc.ClientStream, _ string, limiter limiterhelper.Wrapper) grpc.ClientStream { + return &clientStream{ + ClientStream: cs, + limiter: limiter, + } +} diff --git a/extension/extensionlimiter/limiterhelper/http/httplimiter.go b/extension/extensionlimiter/limiterhelper/http/httplimiter.go new file mode 100644 index 00000000000..94a5402e661 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/http/httplimiter.go @@ -0,0 +1,128 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package httplimiter + +import ( + "context" + "io" + "net/http" + + "go.uber.org/multierr" + + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + "go.opentelemetry.io/collector/extension/extensionmiddleware" + "go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest" +) + +// TODO: network bytes can be implemented after compression middleware, requires +// integration with confighttp. + +func NewClientLimiter(ext extensionlimiter.AnyProvider) (extensionmiddleware.HTTPClient, error) { + wp, err1 := limiterhelper.AnyToWrapperProvider(ext) + rp, err2 := limiterhelper.AnyToRateLimiterProvider(ext) + if err := multierr.Append(err1, err2); err != nil { + return nil, err + } + requestLimiter, err3 := wp.GetWrapper(extensionlimiter.WeightKeyRequestCount) + bytesLimiter, err4 := rp.GetRateLimiter(extensionlimiter.WeightKeyNetworkBytes) + if err := multierr.Append(err3, err4); err != nil { + return nil, err + } + + roundtrip := func(base http.RoundTripper) (http.RoundTripper, error) { + return extensionmiddlewaretest.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + if requestLimiter == nil && bytesLimiter == nil { + // If no limiters are configured, return the base round tripper + return base.RoundTrip(req) + } + + var resp *http.Response + err := requestLimiter.LimitCall( + req.Context(), + 1, + func(_ context.Context) error { + if bytesLimiter != nil && req.Body != nil && req.Body != http.NoBody { + // If bytes are limited, create a limited ReadCloser body. + req.Body = &rateLimitedBody{ + body: req.Body, + limiter: limiterhelper.NewBlockingRateLimiter(bytesLimiter), + ctx: req.Context(), + } + } + var err error + resp, err = base.RoundTrip(req) + return err + }) + return resp, err + }), multierr.Append(err1, err2) + } + return extensionmiddleware.GetHTTPRoundTripperFunc(roundtrip), nil +} + +// rateLimitedBody wraps an http.Request.Body to track bytes and call the rate limiter +type rateLimitedBody struct { + body io.ReadCloser + limiter limiterhelper.BlockingRateLimiter + ctx context.Context +} + +var _ io.ReadCloser = &rateLimitedBody{} + +// Read implements io.Reader interface, limiting bytes as they are read +func (rb *rateLimitedBody) Read(p []byte) (n int, err error) { + n, err = rb.body.Read(p) + if n > 0 { + // Apply rate limiting based on network bytes after they are read + limitErr := rb.limiter.WaitFor(rb.ctx, n) + if limitErr != nil { + // If the rate limiter rejects the bytes, return the error + return n, limitErr // TODO: How to return HTTP 429? + } + } + return n, err +} + +// Close implements io.Closer interface +func (rb *rateLimitedBody) Close() error { + return rb.body.Close() +} + +func NewServerLimiter(ext extensionlimiter.AnyProvider) (extensionmiddleware.HTTPServer, error) { + wp, err1 := limiterhelper.AnyToWrapperProvider(ext) + rp, err2 := limiterhelper.AnyToRateLimiterProvider(ext) + if err := multierr.Append(err1, err2); err != nil { + return nil, err + } + requestLimiter, err3 := wp.GetWrapper(extensionlimiter.WeightKeyRequestCount) + bytesLimiter, err4 := rp.GetRateLimiter(extensionlimiter.WeightKeyNetworkBytes) + if err := multierr.Append(err3, err4); err != nil { + return nil, err + } + + handler := func(base http.Handler) (http.Handler, error) { + if requestLimiter == nil && bytesLimiter == nil { + return base, nil + } + + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _ = requestLimiter.LimitCall( + req.Context(), + 1, + func(_ context.Context) error { + if bytesLimiter != nil && req.Body != nil && req.Body != http.NoBody { + // If bytes are limited, create a limited ReadCloser body. + req.Body = &rateLimitedBody{ + body: req.Body, + limiter: limiterhelper.NewBlockingRateLimiter(bytesLimiter), + ctx: req.Context(), + } + } + base.ServeHTTP(w, req) + return nil + }) + }), nil + } + return extensionmiddleware.GetHTTPHandlerFunc(handler), nil +} diff --git a/extension/extensionlimiter/limiterhelper/middleware.go b/extension/extensionlimiter/limiterhelper/middleware.go new file mode 100644 index 00000000000..a6258607d2e --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/middleware.go @@ -0,0 +1,96 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "errors" + + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +var ( + ErrNotALimiter = errors.New("middleware is not a limiter") + ErrNotARateLimiter = errors.New("middleware cannot implement rate limiter") + ErrLimiterConflict = errors.New("limiter implements both rate and resource-limiters") +) + +// AnyToWrapperProvider returns a limiter wrapper provider from +// middleware. Returns a package-level error if the middleware does +// not implement exactly one of the limiter interfaces (i.e., rate or +// resource). +func AnyToWrapperProvider(ext extensionlimiter.AnyProvider) (WrapperProvider, error) { + return getAny( + ext, + nilError(RateToWrapperProvider), + nilError(ResourceToWrapperProvider), + ) +} + +// AnyToRateLimiterProvider allows a base limiter provider to act as a +// rate limiter provider. This encodes the fact that a resource +// limiter extension cannot be adapted to a rate limiter +// interface. Returns a package-level error if the middleware does not +// implement exactly one of the limiter interfaces (i.e., rate or +// resource). +func AnyToRateLimiterProvider(ext extensionlimiter.AnyProvider) (extensionlimiter.RateLimiterProvider, error) { + return getAny( + ext, + identity[extensionlimiter.RateLimiterProvider], + resourceToRateLimiterError, + ) +} + +// AnyToResourceLimiterProvider allows a base limiter provider to act +// as a resource provider. This enforces that a resource limiter +// extension cannot be adapted to a resource limiter +// interface. Returns a package-level error if the middleware does not +// implement exactly one of the limiter interfaces (i.e., rate or +// resource). +func AnyToResourceLimiterProvider(ext extensionlimiter.AnyProvider) (extensionlimiter.ResourceLimiterProvider, error) { + return getAny( + ext, + nilError(RateToResourceLimiterProvider), + identity[extensionlimiter.ResourceLimiterProvider], + ) +} + +// getAny invokes getProvider if any kind of limiter is detected for +// the given host and middleware configuration. +func getAny[Out any]( + ext extensionlimiter.AnyProvider, + rate func(extensionlimiter.RateLimiterProvider) (Out, error), + resource func(extensionlimiter.ResourceLimiterProvider) (Out, error), +) (Out, error) { + var out Out + res, isResource := ext.(extensionlimiter.ResourceLimiterProvider) + rat, isRate := ext.(extensionlimiter.RateLimiterProvider) + if isResource && isRate { + return out, ErrLimiterConflict + } + if isResource { + return resource(res) + } + if isRate { + return rate(rat) + } + return out, ErrNotALimiter +} + +// identity is a pass-through for the matching provider type. +func identity[T any](lim T) (T, error) { + return lim, nil +} + +// nilError converts an infallible constructor to return a nil error. +func nilError[S, T any](f func(S) T) func(S) (T, error) { + return func(s S) (T, error) { + return f(s), nil + } +} + +// resourceToRateLimiterError represents the impossible conversion +// from resource limiter to rate limiter. +func resourceToRateLimiterError(_ extensionlimiter.ResourceLimiterProvider) (extensionlimiter.RateLimiterProvider, error) { + return nil, ErrNotARateLimiter +} diff --git a/extension/extensionlimiter/limiterhelper/notification.go b/extension/extensionlimiter/limiterhelper/notification.go new file mode 100644 index 00000000000..c54d21a75a8 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/notification.go @@ -0,0 +1,29 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper + +type notification struct { + c chan struct{} +} + +func newNotification() notification { + return notification{c: make(chan struct{})} +} + +func (n *notification) notice() { + close(n.c) +} + +func (n *notification) hasBeen() bool { + select { + case <-n.c: + return true + default: + return false + } +} + +func (n *notification) channel() <-chan struct{} { + return n.c +} diff --git a/extension/extensionlimiter/limiterhelper/rate.go b/extension/extensionlimiter/limiterhelper/rate.go new file mode 100644 index 00000000000..3451d54ea34 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/rate.go @@ -0,0 +1,159 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "context" + "errors" + "time" + + "golang.org/x/time/rate" + + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +var ( + ErrRateLimitExceeded = errors.New("rate limit is saturated") + ErrRateRequestTooLarge = errors.New("rate request exceeds burst size") + ErrRateRequestDeadline = errors.New("rate request exceeds context deadline") +) + +// NewRateLimiter returns an implementation of the rate-limiter +// extension based on logic from x/time/rate. +func NewRateLimiter(frequency float64, burst int) extensionlimiter.RateLimiter { + limit := rate.Limit(frequency) + limiter := rate.NewLimiter(limit, burst) + + return extensionlimiter.NewRateLimiterImpl( + func(ctx context.Context, value int) (extensionlimiter.RateReservation, error) { + // Check if context was canceled. + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + default: + } + + // Check for out-of-range requests. + if value > burst && limit != rate.Inf { + return nil, ErrRateRequestTooLarge + } + + // Call the non-blocking API. + rsv := limiter.ReserveN(time.Now(), value) + if !rsv.OK() { + return nil, ErrRateLimitExceeded + } + + // Compare the wait and deadline. + when := time.Now() + wait := rsv.DelayFrom(when) + if deadline, ok := ctx.Deadline(); ok { + if deadline.Sub(when) < wait { + rsv.Cancel() + return nil, ErrRateRequestDeadline + } + } + + // Return the wait time and cancel function. + return extensionlimiter.NewRateReservationImpl( + func() time.Duration { return wait }, + rsv.Cancel, + ), nil + }, + ) +} + +// BlockingRateLimiter wraps a RateLimiter extension in a blocking +// interface which takes the context deadline into account before +// waiting for the rate allowance. +type BlockingRateLimiter struct { + limiter extensionlimiter.RateLimiter +} + +// NewBlockingRateLimiter returns a blocking wrapper for RateLimiter +// extensions. +func NewBlockingRateLimiter(limiter extensionlimiter.RateLimiter) BlockingRateLimiter { + return BlockingRateLimiter{ + limiter: limiter, + } +} + +// WaitFor blocks the caller until the requested value is allowed by +// the limiter. +func (b BlockingRateLimiter) WaitFor(ctx context.Context, value int) error { + newTimer := func(d time.Duration) (<-chan time.Time, func() bool, func()) { + timer := time.NewTimer(d) + return timer.C, timer.Stop, func() {} + } + return b.waitFor(ctx, value, newTimer) +} + +// timerFunc is a test helper for testing the blocking rate limter. +type timerFunc func(time.Duration) (<-chan time.Time, func() bool, func()) + +// waitFor is a testable form of BlockingRateLimiter.WaitFor. +func (b BlockingRateLimiter) waitFor(ctx context.Context, value int, timer timerFunc) error { + // Reserve from the underlying limiter. + rsv, err := b.limiter.ReserveRate(ctx, value) + if err != nil { + return err + } + + // Wait, if necessary. + delay := rsv.WaitTime() + if delay == 0 { + return nil + } + + ch, stop, advance := timer(delay) + defer stop() + advance() // only has an effect when testing + select { + case <-ch: + // Proceed. Do not cancel. + return nil + + case <-ctx.Done(): + // Context was canceled before we could proceed. + rsv.Cancel() + return context.Cause(ctx) + } +} + +// RateToResourceLimiterProvider allows a rate limiter to act as a +// resource limiter. Note that the opposite direction (i.e., resource +// limter acting as rate limiter) is an invalid configuration. +func RateToResourceLimiterProvider(blimp extensionlimiter.RateLimiterProvider) extensionlimiter.ResourceLimiterProvider { + return extensionlimiter.NewResourceLimiterProviderImpl( + func(weight extensionlimiter.WeightKey, opts ...extensionlimiter.Option) (extensionlimiter.ResourceLimiter, error) { + rlim, err := blimp.GetRateLimiter(weight, opts...) + if err != nil { + return nil, err + } + return extensionlimiter.NewResourceLimiterImpl( + func(ctx context.Context, value int) (extensionlimiter.ResourceReservation, error) { + rsv, err := rlim.ReserveRate(ctx, value) + if err != nil { + return nil, err + } + cch := make(chan struct{}) + timer := time.AfterFunc(rsv.WaitTime(), func() { + close(cch) + }) + return extensionlimiter.NewResourceReservationImpl( + func() <-chan struct{} { return cch }, + func() { + select { + case <-cch: + // The timer fired + default: + rsv.Cancel() + timer.Stop() + } + }, + ), nil + }), nil + }, + ) +} diff --git a/extension/extensionlimiter/limiterhelper/resource.go b/extension/extensionlimiter/limiterhelper/resource.go new file mode 100644 index 00000000000..7623e28a9f8 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/resource.go @@ -0,0 +1,178 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "container/list" + "context" + "errors" + "sync" + + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +var ( + // TODO: was grpccodes.ResourceExhausted + ErrResourceWaitLimit = errors.New("too much waiting data") + + // TODO: was grpccodes.InvalidArgument + ErrResourceSizeLimit = errors.New("request is too large") +) + +// NewResourceLimiter returns an implementation of the +// resource-limiter extension based on a LIFO queue. +// See this [article](https://medium.com/swlh/fifo-considered-harmful-793b76f98374) +// explaining why LIFO is preferred here. +func NewResourceLimiter(admitLimit, waitLimit uint64) extensionlimiter.ResourceLimiter { + return &boundedQueue{ + limitAdmit: admitLimit, + limitWait: waitLimit, + } +} + +var _ extensionlimiter.ResourceLimiter = &boundedQueue{} + +type boundedQueue struct { + limitAdmit uint64 + limitWait uint64 + + // lock protects currentAdmitted, currentWaiting, and waiters + lock sync.Mutex + currentAdmitted uint64 + currentWaiting uint64 + waiters *list.List // of *waiter +} + +// waiter is an item in the BoundedQueue waiters list. +type waiter struct { + notify notification + value int +} + +func (bq *boundedQueue) ReserveResource(ctx context.Context, value int) (extensionlimiter.ResourceReservation, error) { + if uint64(value) > bq.limitAdmit { + return nil, ErrResourceSizeLimit + } + + bq.lock.Lock() + defer bq.lock.Unlock() + + if bq.currentAdmitted+uint64(value) <= bq.limitAdmit { + // the fast success path. + bq.currentAdmitted += uint64(value) + return extensionlimiter.NewResourceReservationImpl( + nil, // No delay + func() { + // There was never a waiter in this + // case, just release and admit waiters. + bq.lock.Lock() + defer bq.lock.Unlock() + + bq.releaseLocked(value) + }, + ), nil + } + + // since we were unable to admit, check if we can wait. + if bq.currentWaiting+uint64(value) > bq.limitWait { + return nil, ErrResourceWaitLimit + } + + // otherwise we need to wait + element := bq.addWaiterLocked(value) + waiter := element.Value.(*waiter) + + return extensionlimiter.NewResourceReservationImpl( + func() <-chan struct{} { + // The caller waits for this notification + // to use the resource. + return waiter.notify.channel() + }, + func() { + // Called when the caller finishes. + bq.lock.Lock() + defer bq.lock.Unlock() + + if waiter.notify.hasBeen() { + // We were also admitted, which can happen + // concurrently with cancellation. Make sure + // to release since no one else will do it. + bq.releaseLocked(value) + } else { + // Remove ourselves from the list of waiters + // so that we can't be admitted in the future. + bq.removeWaiterLocked(value, element) + bq.admitWaitersLocked() + } + }, + ), nil +} + +func (bq *boundedQueue) admitWaitersLocked() { + for bq.waiters.Len() != 0 { + // Ensure there is enough room to admit the next waiter. + element := bq.waiters.Back() + waiter := element.Value.(*waiter) + if bq.currentAdmitted+uint64(waiter.value) > bq.limitAdmit { + // Returning means continuing to wait for the + // most recent arrival to get service by another release. + return + } + + // Release the next waiter and tell it that it has been admitted. + bq.removeWaiterLocked(waiter.value, element) + bq.currentAdmitted += uint64(waiter.value) + + waiter.notify.notice() + } +} + +func (bq *boundedQueue) addWaiterLocked(value int) *list.Element { + bq.currentWaiting += uint64(value) + return bq.waiters.PushBack(&waiter{ + value: value, + notify: newNotification(), + }) +} + +func (bq *boundedQueue) removeWaiterLocked(value int, element *list.Element) { + bq.currentWaiting -= uint64(value) + bq.waiters.Remove(element) +} + +func (bq *boundedQueue) releaseLocked(value int) { + bq.currentAdmitted -= uint64(value) + bq.admitWaitersLocked() +} + +// BlockingResourceLimiter wraps for ResourceLimiter extension in a +// blocking interface which considers the context deadline while +// waiting for the resource. +type BlockingResourceLimiter struct { + limiter extensionlimiter.ResourceLimiter +} + +// NewBlockingResourceLimiter returns a blocking wrapper for +// ResourceLimiter extensions. +func NewBlockingResourceLimiter(limiter extensionlimiter.ResourceLimiter) BlockingResourceLimiter { + return BlockingResourceLimiter{ + limiter: limiter, + } +} + +// WaitFor blocks the caller until the requested value is allowed by +// the limiter. +func (b BlockingResourceLimiter) WaitFor(ctx context.Context, value int) (extensionlimiter.ReleaseFunc, error) { + rsv, err := b.limiter.ReserveResource(ctx, value) + if err != nil { + return func() {}, err + } + select { + case <-ctx.Done(): + rsv.Release() + return func() {}, context.Cause(ctx) + case <-rsv.Delay(): + return rsv.Release, nil + } +} diff --git a/extension/extensionlimiter/limiterhelper/wrapper.go b/extension/extensionlimiter/limiterhelper/wrapper.go new file mode 100644 index 00000000000..aebcc3f9d75 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/wrapper.go @@ -0,0 +1,110 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "context" + + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +// Wrapper is a general-purpose interface for limiter consumers +// to limit resources with use of a callback. This is the simplest +// form of rate limiting interface from a callers perspective. If the +// caller is a pipeline component, consider using a consumer-oriented +// limiterhelper (e.g., limiterhelper.NewLimitedLogs) to simplify +// construction of this interface. +// +// A wrapped limiter is either a RateLimiter or ResourceLimiter +// interface. Wrappers can be constructed from either of the +// underlying limiters and their corresponding providers. Usually +// configmiddleware or limiterhelper is responsible for constructing +// the correct wrapper from these two kinds of limiter; users will use +// this interface consistently. +type Wrapper interface { + // LimitCall applies the limiter and with the rate or resource + // granted makes a scoped call, returning success or an error + // from either the limiter or the enclosed callback. + // + // The `call` parameter must be non-nil. + LimitCall(ctx context.Context, weight int, call func(ctx context.Context) error) error +} + +// LimitCallFunc is a functional way to build Wrappers. +type LimitCallFunc func(context.Context, int, func(ctx context.Context) error) error + +// LimitCall implements part of the Wrapper interface. +func (f LimitCallFunc) LimitCall(ctx context.Context, value int, call func(ctx context.Context) error) error { + if f == nil { + return call(ctx) + } + return f(ctx, value, call) +} + +// wrapperImpl is a functional Wrapper object. The zero state is a +// no-op. +type wrapperImpl struct { + LimitCallFunc +} + +var _ Wrapper = wrapperImpl{} + +// NewWrapperImpl returns a functional implementation of +// Wrapper. Use a nil argument for the no-op implementation. +func NewWrapperImpl(f LimitCallFunc) Wrapper { + return wrapperImpl{ + LimitCallFunc: f, + } +} + +// ResourceToWrapperProvider constructs a +// WrapperProvider for a resource limiter extension. +func ResourceToWrapperProvider(rp extensionlimiter.ResourceLimiterProvider) WrapperProvider { + return NewWrapperProviderImpl( + func(key extensionlimiter.WeightKey, opts ...extensionlimiter.Option) (Wrapper, error) { + lim, err := rp.GetResourceLimiter(key, opts...) + if err != nil { + return nil, err + } + if lim == nil { + return NewWrapperImpl(nil), nil + } + blocking := NewBlockingResourceLimiter(lim) + return NewWrapperImpl( + func(ctx context.Context, value int, call func(context.Context) error) error { + release, err := blocking.WaitFor(ctx, value) + if err != nil { + return err + } + defer release() + return call(ctx) + }, + ), nil + }) +} + +// RateToWrapperProvider constructs a WrapperProvider +// for a rate limiter extension. +func RateToWrapperProvider(rp extensionlimiter.RateLimiterProvider) WrapperProvider { + return NewWrapperProviderImpl( + func(key extensionlimiter.WeightKey, opts ...extensionlimiter.Option) (Wrapper, error) { + lim, err := rp.GetRateLimiter(key, opts...) + if err != nil { + return nil, err + } + if lim == nil { + return NewWrapperImpl(nil), nil + } + blocking := NewBlockingRateLimiter(lim) + return NewWrapperImpl( + func(ctx context.Context, value int, call func(context.Context) error) error { + if err := blocking.WaitFor(ctx, value); err != nil { + return err + } + return call(ctx) + }, + ), nil + }, + ) +} diff --git a/extension/extensionlimiter/limiterhelper/wrapper_provider.go b/extension/extensionlimiter/limiterhelper/wrapper_provider.go new file mode 100644 index 00000000000..d2241389e68 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/wrapper_provider.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +// WrapperProvider follows the provider pattern for +// the Wrapper type +type WrapperProvider interface { + GetWrapper(extensionlimiter.WeightKey, ...extensionlimiter.Option) (Wrapper, error) +} + +// GetWrapperFunc is an easy way to build GetWrapper functions. +type GetWrapperFunc func(extensionlimiter.WeightKey, ...extensionlimiter.Option) (Wrapper, error) + +// GetWrapper implements WrapperProvider. +func (f GetWrapperFunc) GetWrapper(key extensionlimiter.WeightKey, opts ...extensionlimiter.Option) (Wrapper, error) { + if f == nil { + return NewWrapperImpl(nil), nil + } + return f(key, opts...) +} + +type limiterWrapperProviderImpl struct { + GetWrapperFunc +} + +var _ WrapperProvider = limiterWrapperProviderImpl{} + +// NewWrapperProviderImpl returns a functional implementation of +// WrapperProvider. Use a nil argument for the no-op implementation. +func NewWrapperProviderImpl(f GetWrapperFunc) WrapperProvider { + return limiterWrapperProviderImpl{ + GetWrapperFunc: f, + } +} diff --git a/extension/extensionlimiter/option.go b/extension/extensionlimiter/option.go new file mode 100644 index 00000000000..16ac275b7f5 --- /dev/null +++ b/extension/extensionlimiter/option.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// Option is passed to limiter providers. +// +// NOTE: For data-specific or tenant-specific limits we will extend +// providers with Options and add a Config type, but none are +// supported yet and this PR contains only interfaces, not need for +// options in core repository components. +type Option interface { + apply() +} diff --git a/extension/extensionlimiter/rate_limiter.go b/extension/extensionlimiter/rate_limiter.go new file mode 100644 index 00000000000..05c61f36bfe --- /dev/null +++ b/extension/extensionlimiter/rate_limiter.go @@ -0,0 +1,57 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +import ( + "context" +) + +// RateLimiter is an interface that an implementation makes available +// to apply time-based limits on quantities such as the number of +// bytes or items per second. +// +// This is a relatively low-level interface. Callers that can use a +// LimiterWrapper should choose that interface instead. This interface +// is meant for direct use only in special cases where control flow +// cannot be easily scoped to a callback, for example inside +// middleware (e.g., grpc.StatsHandler). +// +// See the README for more recommendations. +type RateLimiter interface { + // ReserveRate is modeled on pkg.go.dev/golang.org/x/time/rate#Limiter.ReserveN + // + // This is a non-blocking interface; use this interface for + // callers that cannot be blocked but will instead schedule a + // resume after Delay(). The context is provided for access to + // instrumentation and client metadata; the Context deadline + // is not used, should be considered by the caller. + ReserveRate(context.Context, int) (RateReservation, error) +} + +// ReserveRateFunc is a functional way to construct ReserveRate functions. +type ReserveRateFunc func(context.Context, int) (RateReservation, error) + +// Reserve implements part of the RateLimiter interface. +func (f ReserveRateFunc) ReserveRate(ctx context.Context, value int) (RateReservation, error) { + if f == nil { + return NewRateReservationImpl(nil, nil), nil + } + return f(ctx, value) +} + +// rateLimiterImpl is a functional RateLimiter object. The zero state +// is a no-op. +type rateLimiterImpl struct { + ReserveRateFunc +} + +var _ RateLimiter = rateLimiterImpl{} + +// NewRateLimiterImpl returns a functional implementation of +// RateLimiter. Use a nil argument for the no-op implementation. +func NewRateLimiterImpl(f ReserveRateFunc) RateLimiter { + return rateLimiterImpl{ + ReserveRateFunc: f, + } +} diff --git a/extension/extensionlimiter/rate_limiter_provider.go b/extension/extensionlimiter/rate_limiter_provider.go new file mode 100644 index 00000000000..97daee44dc0 --- /dev/null +++ b/extension/extensionlimiter/rate_limiter_provider.go @@ -0,0 +1,46 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// RateLimiterProvider is a provider for rate limiters. +// +// Limiter implementations will implement this or the +// ResourceLimiterProvider interface, but MUST not implement both. +// Limiters are covered by configmiddleware configuration, which is +// able to construct LimiterWrappers from these providers. +type RateLimiterProvider interface { + // AnyProvider is provided by embedding AnyProviderImpl. + AnyProvider + + // GetRateLimiter returns a rate limiter for a weight key. + GetRateLimiter(WeightKey, ...Option) (RateLimiter, error) +} + +// GetRateLimiterFunc is a functional way to construct GetRateLimiter +// functions. +type GetRateLimiterFunc func(WeightKey, ...Option) (RateLimiter, error) + +// RateLimiter implements RateLimiterProvider. +func (f GetRateLimiterFunc) GetRateLimiter(key WeightKey, opts ...Option) (RateLimiter, error) { + if f == nil { + return NewRateLimiterImpl(nil), nil + } + return f(key, opts...) +} + +type rateLimiterProviderImpl struct { + AnyProviderImpl + + GetRateLimiterFunc +} + +var _ RateLimiterProvider = rateLimiterProviderImpl{} + +// NewRateLimiterProviderImpl returns a functional implementation of +// RateLimiterProvider. Use a nil argument for the no-op implementation. +func NewRateLimiterProviderImpl(f GetRateLimiterFunc) RateLimiterProvider { + return rateLimiterProviderImpl{ + GetRateLimiterFunc: f, + } +} diff --git a/extension/extensionlimiter/rate_reservation.go b/extension/extensionlimiter/rate_reservation.go new file mode 100644 index 00000000000..472e7b5efd1 --- /dev/null +++ b/extension/extensionlimiter/rate_reservation.go @@ -0,0 +1,61 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +import ( + "time" +) + +// RateReservation is modeled on pkg.go.dev/golang.org/x/time/rate#Reservation +type RateReservation interface { + // WaitTime returns the duration until this reservation may + // proceed. A typical implementation uses Reservation.DelayFrom(time.Now()), + // and callers typically/ use time.After or time.Timer to implement a delay. + WaitTime() time.Duration + + // Cancel cancels the reservation before it is used. A typical + // implementation uses Reservation.CancelAt(time.Now()). + Cancel() +} + +// WaitTimeFunc is a functional way to construct WaitTime functions. +type WaitTimeFunc func() time.Duration + +// WaitTime implements part of Reservation. +func (f WaitTimeFunc) WaitTime() time.Duration { + if f == nil { + return 0 + } + return f.WaitTime() +} + +// CancelFunc is a functional way to construct Cancel functions. +type CancelFunc func() + +// Reserve implements part of the RateReserveer interface. +func (f CancelFunc) Cancel() { + if f == nil { + return + } + f.Cancel() +} + +// rateReservationImpl is a struct that implements RateReservation. +// The zero state is a no-op. +type rateReservationImpl struct { + WaitTimeFunc + CancelFunc +} + +var _ RateReservation = rateReservationImpl{} + +// NewRateReservationImpl returns a functional implementation of +// RateReservation. Use a nil argument for the no-op implementation. +func NewRateReservationImpl(wf WaitTimeFunc, cf CancelFunc) RateReservation { + return rateReservationImpl{ + WaitTimeFunc: wf, + CancelFunc: cf, + } +} + diff --git a/extension/extensionlimiter/resource_limiter.go b/extension/extensionlimiter/resource_limiter.go new file mode 100644 index 00000000000..7e1c743a336 --- /dev/null +++ b/extension/extensionlimiter/resource_limiter.go @@ -0,0 +1,60 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +import ( + "context" +) + +// ResourceLimiter is an interface that an implementation makes +// available to apply physical limits on quantities such as the number +// of concurrent requests or amount of memory in use. +// +// This is a relatively low-level interface. Callers that can use a +// LimiterWrapper should choose that interface instead. This +// interface is meant for direct use only in special cases where +// control flow is not scoped to a callback, for example in a +// streaming receiver where a limiter might be Acquired in the body of +// Send() and released prior to a corresponding Recv() (e.g., +// OTel-Arrow receiver). +// +// See the README for more recommendations. +type ResourceLimiter interface { + // ReserveRate is modeled on pkg.go.dev/golang.org/x/time/rate#Limiter.ReserveN, + // without the time dimension. + // + // This is a non-blocking interface; use this interface for + // callers that cannot be blocked but will instead schedule a + // resume after Delay(). The context is provided for + // access to instrumentation and client metadata; the Context + // deadline is not used. + ReserveResource(context.Context, int) (ResourceReservation, error) +} + +// ReserveResourceFunc is a functional way to construct ReserveResource interface methods. +type ReserveResourceFunc func(ctx context.Context, value int) (ResourceReservation, error) + +// ReserveResource implements a ReserveResource interface method. +func (f ReserveResourceFunc) ReserveResource(ctx context.Context, value int) (ResourceReservation, error) { + if f == nil { + return NewResourceReservationImpl(nil, nil), nil + } + return f(ctx, value) +} + +// resourceLimiterImpl is a functional ResourceLimiter object. The zero state +// is a no-op. +type resourceLimiterImpl struct { + ReserveResourceFunc +} + +var _ ResourceLimiter = resourceLimiterImpl{} + +// NewResourceLimiterImpl returns a functional implementation of +// ResourceLimiter. Use a nil argument for the no-op implementation. +func NewResourceLimiterImpl(f ReserveResourceFunc) ResourceLimiter { + return resourceLimiterImpl{ + ReserveResourceFunc: f, + } +} diff --git a/extension/extensionlimiter/resource_limiter_provider.go b/extension/extensionlimiter/resource_limiter_provider.go new file mode 100644 index 00000000000..875f68b529f --- /dev/null +++ b/extension/extensionlimiter/resource_limiter_provider.go @@ -0,0 +1,41 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// ResourceLimiterProvider is a provider for resource limiters. +// +// Limiter implementations will implement this or the +// RateLimiterProvider interface, but MUST not implement both. +// Limiters are covered by configmiddleware configuration, which +// is able to construct LimiterWrappers from these providers. +type ResourceLimiterProvider interface { + GetResourceLimiter(WeightKey, ...Option) (ResourceLimiter, error) +} + +// GetResourceLimiterFunc is a functional way to construct +// GetResourceLimiter functions. +type GetResourceLimiterFunc func(WeightKey, ...Option) (ResourceLimiter, error) + +// GetResourceLimiter implements part of ResourceLimiterProvider. +func (f GetResourceLimiterFunc) GetResourceLimiter(key WeightKey, opts ...Option) (ResourceLimiter, error) { + if f == nil { + return NewResourceLimiterImpl(nil), nil + } + return f(key, opts...) +} + + +type resourceLimiterProviderImpl struct { + GetResourceLimiterFunc +} + +var _ ResourceLimiterProvider = resourceLimiterProviderImpl{} + +// NewResourceLimiterProviderImpl returns a functional implementation of +// ResourceLimiterProvider. Use a nil argument for the no-op implementation. +func NewResourceLimiterProviderImpl(f GetResourceLimiterFunc) ResourceLimiterProvider { + return resourceLimiterProviderImpl{ + GetResourceLimiterFunc: f, + } +} diff --git a/extension/extensionlimiter/resource_reservation.go b/extension/extensionlimiter/resource_reservation.go new file mode 100644 index 00000000000..9d447628f4e --- /dev/null +++ b/extension/extensionlimiter/resource_reservation.go @@ -0,0 +1,68 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// ResourceReservation is modeled on pkg.go.dev/golang.org/x/time/rate#Reservation +// without the time dimension. +type ResourceReservation interface { + // Delay returns a channel that will be closed when the + // reservation request is granted. This resembles a context + // Done channel. + Delay() <-chan struct{} + + // Release is called after finishing with the reservation, + // whether the delay has been reached or not. + Release() +} + +// ReleaseFunc is called when resources have been released after use. +// +// RelaseFunc values are never nil values, even in the error case for +// safety. Users typically will immediately defer a call to this. +type ReleaseFunc func() + +// Release calls this function. +func (f ReleaseFunc) Release() { + if f == nil { + return + } + f() +} + +// DelayFunc returns a channel that is closed when the request is +// permitted to go ahead. +type DelayFunc func() <-chan struct{} + +// Delay calls this function. +func (f DelayFunc) Delay() <-chan struct{} { + if f == nil { + return immediateChan + } + return f() +} + +// immediateChan is a singleton channel, already closed, used when a DelayFunc is nil. +var immediateChan = func() <-chan struct{} { + ic := make(chan struct{}) + close(ic) + return ic +}() + +// resourceReservationImpl is a struct that implements ResourceReservation. +// The zero state is a no-op. +type resourceReservationImpl struct { + DelayFunc + ReleaseFunc +} + +var _ ResourceReservation = resourceReservationImpl{} + +// NewResourceReservationImpl returns a functional implementation of +// ResourceReservation. Use a nil argument for the no-op implementation. +func NewResourceReservationImpl(df DelayFunc, rf ReleaseFunc) ResourceReservation { + return resourceReservationImpl{ + DelayFunc: df, + ReleaseFunc: rf, + } +} diff --git a/extension/extensionlimiter/weight.go b/extension/extensionlimiter/weight.go new file mode 100644 index 00000000000..51b84fb286b --- /dev/null +++ b/extension/extensionlimiter/weight.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +// WeightKey is an enum type for common rate limits. The +// StandardAllKeys, StandardMiddlewareKeys, and +// StandardNotMiddlewareKeys methods return the list of middleware +// keys that can be automatically configured through middleware and +// not. +type WeightKey string + +// Predefined weight keys for common rate limits. This is not meant +// to be a closed set, new weight keys may be added in the future, +// possibly to restrict other kinds of event (e.g., auths, retries). +// +// Providers should return errors when they do not recognize a weight +// key. +const ( + // WeightKeyNetworkBytes is for network bytes. This is + // typically used with rate limiters. + WeightKeyNetworkBytes WeightKey = "network_bytes" + + // WeightKeyRequestCount can be used to limit the rate or + // total concurrent number of requests (i.e., pipeline data + // objects). This is typically used with both rate and + // resource limiters. + WeightKeyRequestCount WeightKey = "request_count" + + // WeightKeyRequestItems can be used to limit the rate or + // total concurrent number of items (log records, metric data + // points, spans, profiles). This is typically used with both + // rate and resource limiters. + WeightKeyRequestItems WeightKey = "request_items" + + // WeightKeyRequestBytes is typically used with ResourceLimiters + // for limiting active memory usage. + WeightKeyRequestBytes WeightKey = "request_bytes" +) diff --git a/extension/memorylimiterextension/go.mod b/extension/memorylimiterextension/go.mod index 7b3cf34542e..6fb07f741a2 100644 --- a/extension/memorylimiterextension/go.mod +++ b/extension/memorylimiterextension/go.mod @@ -8,6 +8,7 @@ require ( go.opentelemetry.io/collector/component/componenttest v0.128.0 go.opentelemetry.io/collector/confmap v1.34.0 go.opentelemetry.io/collector/extension v1.34.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 go.opentelemetry.io/collector/extension/extensiontest v0.128.0 go.opentelemetry.io/collector/internal/memorylimiter v0.128.0 go.uber.org/goleak v1.3.0 @@ -78,3 +79,15 @@ replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../extensionmiddleware + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer => ../../consumer diff --git a/extension/memorylimiterextension/memorylimiter.go b/extension/memorylimiterextension/memorylimiter.go index 4986243ce10..f9a92eb3fa4 100644 --- a/extension/memorylimiterextension/memorylimiter.go +++ b/extension/memorylimiterextension/memorylimiter.go @@ -5,17 +5,27 @@ package memorylimiterextension // import "go.opentelemetry.io/collector/extensio import ( "context" + "errors" "go.uber.org/zap" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension/extensionlimiter" "go.opentelemetry.io/collector/internal/memorylimiter" ) +var ( + ErrMustRefuse = errors.New("system is near memory limit") +) + type memoryLimiterExtension struct { + extensionlimiter.AnyProviderImpl + memLimiter *memorylimiter.MemoryLimiter } +var _ extensionlimiter.RateLimiterProvider = &memoryLimiterExtension{} + // newMemoryLimiter returns a new memorylimiter extension. func newMemoryLimiter(cfg *Config, logger *zap.Logger) (*memoryLimiterExtension, error) { ml, err := memorylimiter.NewMemoryLimiter(cfg, logger) @@ -34,6 +44,21 @@ func (ml *memoryLimiterExtension) Shutdown(ctx context.Context) error { return ml.memLimiter.Shutdown(ctx) } +// GetRateLimiter implements extensionlimiter.RateLimiterProvider. +// Note that this extension ignores the weight/key, the context, the +// and the options. +func (ml *memoryLimiterExtension) GetRateLimiter( + _ extensionlimiter.WeightKey, + _ ...extensionlimiter.Option, +) (extensionlimiter.RateLimiter, error) { + return extensionlimiter.ReserveRateFunc(func(_ context.Context, _ int) (extensionlimiter.RateReservation, error) { + if ml.MustRefuse() { + return nil, ErrMustRefuse + } + return extensionlimiter.NewRateReservationImpl(nil, nil), nil + }), nil +} + // MustRefuse returns if the caller should deny because memory has reached it's configured limits func (ml *memoryLimiterExtension) MustRefuse() bool { return ml.memLimiter.MustRefuse() diff --git a/extension/zpagesextension/go.mod b/extension/zpagesextension/go.mod index 6e2262dd6d6..1df3e96ac45 100644 --- a/extension/zpagesextension/go.mod +++ b/extension/zpagesextension/go.mod @@ -34,12 +34,15 @@ require ( github.com/google/go-tpm v0.9.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect github.com/knadh/koanf/providers/confmap v1.0.0 // indirect github.com/knadh/koanf/v2 v2.2.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/cors v1.11.1 // indirect @@ -49,11 +52,16 @@ require ( go.opentelemetry.io/collector/config/configmiddleware v0.128.0 // indirect go.opentelemetry.io/collector/config/configopaque v1.34.0 // indirect go.opentelemetry.io/collector/config/configtls v1.34.0 // indirect + go.opentelemetry.io/collector/consumer v1.34.0 // indirect + go.opentelemetry.io/collector/consumer/xconsumer v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect go.opentelemetry.io/collector/pdata v1.34.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect go.opentelemetry.io/collector/pipeline v0.128.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect @@ -123,3 +131,9 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../extens replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile diff --git a/extension/zpagesextension/go.sum b/extension/zpagesextension/go.sum index d4f126c8fdb..7fbfad7ffea 100644 --- a/extension/zpagesextension/go.sum +++ b/extension/zpagesextension/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -30,6 +31,7 @@ github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/go-tpm-tools v0.4.4 h1:oiQfAIkc6xTy9Fl5NKTeTJkBTlXdHsxAofmQyxBKY98= github.com/google/go-tpm-tools v0.4.4/go.mod h1:T8jXkp2s+eltnCDIsXR84/MTcVU9Ja7bh3Mit0pa4AY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= @@ -54,6 +56,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -66,6 +69,8 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/e2e/go.mod b/internal/e2e/go.mod index 697c99736c2..a4715bcb3cc 100644 --- a/internal/e2e/go.mod +++ b/internal/e2e/go.mod @@ -95,7 +95,9 @@ require ( go.opentelemetry.io/collector/exporter/xexporter v0.128.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensioncapabilities v0.128.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/extension/extensiontest v0.128.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect @@ -204,8 +206,6 @@ replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/config/configtelemetry => ../../config/configtelemetry -replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest replace go.opentelemetry.io/collector/client => ../../client @@ -274,4 +274,8 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../ext replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../../pdata/xpdata diff --git a/otelcol/go.mod b/otelcol/go.mod index 9da383f4d73..19b056ba376 100644 --- a/otelcol/go.mod +++ b/otelcol/go.mod @@ -174,8 +174,6 @@ replace go.opentelemetry.io/collector/config/configtls => ../config/configtls replace go.opentelemetry.io/collector/config/configopaque => ../config/configopaque -replace go.opentelemetry.io/collector/consumer/xconsumer => ../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../consumer/consumertest replace go.opentelemetry.io/collector/client => ../client @@ -224,4 +222,8 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../extens replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/consumer/xconsumer => ../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../pdata/xpdata diff --git a/otelcol/otelcoltest/go.mod b/otelcol/otelcoltest/go.mod index a9aa8ae91be..2cbea14d911 100644 --- a/otelcol/otelcoltest/go.mod +++ b/otelcol/otelcoltest/go.mod @@ -161,8 +161,6 @@ replace go.opentelemetry.io/collector/extension => ../../extension replace go.opentelemetry.io/collector/exporter => ../../exporter -replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest replace go.opentelemetry.io/collector/component/componentstatus => ../../component/componentstatus @@ -233,4 +231,8 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmid replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../../pdata/xpdata diff --git a/receiver/otlpreceiver/config.go b/receiver/otlpreceiver/config.go index 801595efc2a..c04feeb6ff7 100644 --- a/receiver/otlpreceiver/config.go +++ b/receiver/otlpreceiver/config.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configoptional" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper/consumerlimiter" ) type SanitizedURLPath string @@ -62,6 +63,9 @@ type Protocols struct { type Config struct { // Protocols is the configuration for the supported protocols, currently gRPC and HTTP (Proto and JSON). Protocols `mapstructure:"protocols"` + + // LimiterConfig allows applying limiter extensions for request count, items, and bytes. + consumerlimiter.LimiterConfig `mapstructure:"limiters"` } var _ component.Config = (*Config)(nil) diff --git a/receiver/otlpreceiver/config_test.go b/receiver/otlpreceiver/config_test.go index 8247a7bb882..5b34e32eeee 100644 --- a/receiver/otlpreceiver/config_test.go +++ b/receiver/otlpreceiver/config_test.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/collector/config/configauth" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configmiddleware" "go.opentelemetry.io/collector/config/confignet" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configoptional" @@ -97,6 +98,27 @@ func TestUnmarshalConfigOnlyHTTPEmptyMap(t *testing.T) { assert.Equal(t, defaultOnlyHTTP, cfg) } +func TestUnmarshalLimiterConfig(t *testing.T) { + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "limiters.yaml")) + require.NoError(t, err) + factory := NewFactory() + cfg := factory.CreateDefaultConfig().(*Config) + require.NoError(t, cm.Unmarshal(&cfg)) + + assert.Equal(t, cfg.Protocols.GRPC.Get().Middlewares, []configmiddleware.Config{ + configmiddleware.Config(component.MustNewIDWithName("one", "a")), + configmiddleware.Config(component.MustNewID("two")), + }) + assert.Equal(t, cfg.Protocols.HTTP.Get().ServerConfig.Middlewares, []configmiddleware.Config{ + configmiddleware.Config(component.MustNewIDWithName("three", "b")), + configmiddleware.Config(component.MustNewID("four")), + }) + assert.Equal(t, cfg.LimiterConfig.RequestBytes, component.MustNewIDWithName("five", "c")) + assert.Equal(t, cfg.LimiterConfig.RequestItems, component.MustNewID("six")) + assert.Equal(t, cfg.LimiterConfig.RequestCount, component.MustNewID("seven")) + +} + func TestUnmarshalConfig(t *testing.T) { cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) require.NoError(t, err) diff --git a/receiver/otlpreceiver/factory.go b/receiver/otlpreceiver/factory.go index 6564ea4e8f0..d78ea5cd3a6 100644 --- a/receiver/otlpreceiver/factory.go +++ b/receiver/otlpreceiver/factory.go @@ -17,6 +17,7 @@ import ( "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/otlpreceiver/internal/metadata" "go.opentelemetry.io/collector/receiver/xreceiver" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper/consumerlimiter" ) const ( @@ -28,13 +29,18 @@ const ( // NewFactory creates a new OTLP receiver factory. func NewFactory() receiver.Factory { - return xreceiver.NewFactory( - metadata.Type, - createDefaultConfig, - xreceiver.WithTraces(createTraces, metadata.TracesStability), - xreceiver.WithMetrics(createMetrics, metadata.MetricsStability), - xreceiver.WithLogs(createLog, metadata.LogsStability), - xreceiver.WithProfiles(createProfiles, metadata.ProfilesStability), + return consumerlimiter.NewLimitedFactory( + xreceiver.NewFactory( + metadata.Type, + createDefaultConfig, + xreceiver.WithTraces(createTraces, metadata.TracesStability), + xreceiver.WithMetrics(createMetrics, metadata.MetricsStability), + xreceiver.WithLogs(createLog, metadata.LogsStability), + xreceiver.WithProfiles(createProfiles, metadata.ProfilesStability), + ), + func(cfg component.Config) consumerlimiter.LimiterConfig { + return cfg.(*Config).LimiterConfig + }, ) } diff --git a/receiver/otlpreceiver/go.mod b/receiver/otlpreceiver/go.mod index f3e763ef481..fc59e0fad44 100644 --- a/receiver/otlpreceiver/go.mod +++ b/receiver/otlpreceiver/go.mod @@ -13,6 +13,7 @@ require ( go.opentelemetry.io/collector/config/configauth v0.128.0 go.opentelemetry.io/collector/config/configgrpc v0.128.0 go.opentelemetry.io/collector/config/confighttp v0.128.0 + go.opentelemetry.io/collector/config/configmiddleware v0.128.0 go.opentelemetry.io/collector/config/confignet v1.34.0 go.opentelemetry.io/collector/config/configopaque v1.34.0 go.opentelemetry.io/collector/config/configoptional v0.128.0 @@ -23,6 +24,7 @@ require ( go.opentelemetry.io/collector/consumer/consumererror v0.128.0 go.opentelemetry.io/collector/consumer/consumertest v0.128.0 go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 go.opentelemetry.io/collector/internal/sharedcomponent v0.128.0 go.opentelemetry.io/collector/internal/telemetry v0.128.0 go.opentelemetry.io/collector/pdata v1.34.0 @@ -69,9 +71,10 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/collector/client v1.34.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.34.0 // indirect - go.opentelemetry.io/collector/config/configmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension v1.34.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/collector/featuregate v1.34.0 // indirect go.opentelemetry.io/collector/pipeline v0.128.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect @@ -132,8 +135,6 @@ replace go.opentelemetry.io/collector/consumer => ../../consumer replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile -replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest replace go.opentelemetry.io/collector/client => ../../client @@ -166,3 +167,7 @@ replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../ext replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter diff --git a/receiver/otlpreceiver/testdata/limiters.yaml b/receiver/otlpreceiver/testdata/limiters.yaml new file mode 100644 index 00000000000..9cdb34f3845 --- /dev/null +++ b/receiver/otlpreceiver/testdata/limiters.yaml @@ -0,0 +1,15 @@ +# The following entry initializes the default OTLP receiver. +# The full name of this receiver is `otlp` and can be referenced in pipelines by 'otlp'. +protocols: + grpc: + middleware: + - one/a + - two + http: + middleware: + - three/b + - four +limiters: + request_bytes: five/c + request_items: six + request_count: seven diff --git a/receiver/receiver.go b/receiver/receiver.go index f2aa1fe2c58..2e581d9bb4f 100644 --- a/receiver/receiver.go +++ b/receiver/receiver.go @@ -93,14 +93,14 @@ type Factory interface { // FactoryOption apply changes to Factory. type FactoryOption interface { // applyOption applies the option. - applyOption(o *factory) + applyOption(o *factoryImpl, cfgType component.Type) } // factoryOptionFunc is an FactoryOption created through a function. -type factoryOptionFunc func(*factory) +type factoryOptionFunc func(*factoryImpl, component.Type) -func (f factoryOptionFunc) applyOption(o *factory) { - f(o) +func (f factoryOptionFunc) applyOption(o *factoryImpl, cfgType component.Type) { + f(o, cfgType) } // CreateTracesFunc is the equivalent of Factory.CreateTraces. @@ -112,103 +112,137 @@ type CreateMetricsFunc func(context.Context, Settings, component.Config, consume // CreateLogsFunc is the equivalent of Factory.CreateLogs. type CreateLogsFunc func(context.Context, Settings, component.Config, consumer.Logs) (Logs, error) -type factory struct { - cfgType component.Type - component.CreateDefaultConfigFunc - createTracesFunc CreateTracesFunc - tracesStabilityLevel component.StabilityLevel - createMetricsFunc CreateMetricsFunc - metricsStabilityLevel component.StabilityLevel - createLogsFunc CreateLogsFunc - logsStabilityLevel component.StabilityLevel -} +// TracesStabilityFunc is a functional way to construct Factory implementations. +type TracesStabilityFunc func() component.StabilityLevel -func (f *factory) Type() component.Type { - return f.cfgType -} +// MetricsStabilityFunc is a functional way to construct Factory implementations. +type MetricsStabilityFunc func() component.StabilityLevel -func (f *factory) unexportedFactoryFunc() {} +// LogsStabilityFunc is a functional way to construct Factory implementations. +type LogsStabilityFunc func() component.StabilityLevel -func (f *factory) TracesStability() component.StabilityLevel { - return f.tracesStabilityLevel +type factoryImpl struct { + component.Factory + CreateTracesFunc + TracesStabilityFunc + CreateMetricsFunc + MetricsStabilityFunc + CreateLogsFunc + LogsStabilityFunc } -func (f *factory) MetricsStability() component.StabilityLevel { - return f.metricsStabilityLevel -} +var _ Factory = factoryImpl{} -func (f *factory) LogsStability() component.StabilityLevel { - return f.logsStabilityLevel -} +func (f factoryImpl) unexportedFactoryFunc() {} -func (f *factory) CreateTraces(ctx context.Context, set Settings, cfg component.Config, next consumer.Traces) (Traces, error) { - if f.createTracesFunc == nil { - return nil, pipeline.ErrSignalNotSupported +func (f TracesStabilityFunc) TracesStability() component.StabilityLevel { + if f == nil { + return component.StabilityLevelUndefined } + return f() +} - if set.ID.Type() != f.Type() { - return nil, internal.ErrIDMismatch(set.ID, f.Type()) +func (f MetricsStabilityFunc) MetricsStability() component.StabilityLevel { + if f == nil { + return component.StabilityLevelUndefined } + return f() +} - return f.createTracesFunc(ctx, set, cfg, next) +func (f LogsStabilityFunc) LogsStability() component.StabilityLevel { + if f == nil { + return component.StabilityLevelUndefined + } + return f() } -func (f *factory) CreateMetrics(ctx context.Context, set Settings, cfg component.Config, next consumer.Metrics) (Metrics, error) { - if f.createMetricsFunc == nil { - return nil, pipeline.ErrSignalNotSupported +type creator[A, B any] func(ctx context.Context, set Settings, cfg component.Config, next A) (B, error) + +func typeChecked[A, B any](cf creator[A, B], cfgType component.Type) creator[A, B] { + return func(ctx context.Context, set Settings, cfg component.Config, next A) (B, error) { + if set.ID.Type() != cfgType { + var zero B + return zero, internal.ErrIDMismatch(set.ID, cfgType) + } + return cf(ctx, set, cfg, next) } +} - if set.ID.Type() != f.Type() { - return nil, internal.ErrIDMismatch(set.ID, f.Type()) +func (f CreateTracesFunc) CreateTraces(ctx context.Context, set Settings, cfg component.Config, next consumer.Traces) (Traces, error) { + if f == nil { + return nil, pipeline.ErrSignalNotSupported } - return f.createMetricsFunc(ctx, set, cfg, next) + return f(ctx, set, cfg, next) } -func (f *factory) CreateLogs(ctx context.Context, set Settings, cfg component.Config, next consumer.Logs) (Logs, error) { - if f.createLogsFunc == nil { +func (f CreateMetricsFunc) CreateMetrics(ctx context.Context, set Settings, cfg component.Config, next consumer.Metrics) (Metrics, error) { + if f == nil { return nil, pipeline.ErrSignalNotSupported } - if set.ID.Type() != f.Type() { - return nil, internal.ErrIDMismatch(set.ID, f.Type()) + return f(ctx, set, cfg, next) +} + +func (f CreateLogsFunc) CreateLogs(ctx context.Context, set Settings, cfg component.Config, next consumer.Logs) (Logs, error) { + if f == nil { + return nil, pipeline.ErrSignalNotSupported } - return f.createLogsFunc(ctx, set, cfg, next) + return f(ctx, set, cfg, next) } // WithTraces overrides the default "error not supported" implementation for Factory.CreateTraces and the default "undefined" stability level. func WithTraces(createTraces CreateTracesFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factory) { - o.tracesStabilityLevel = sl - o.createTracesFunc = createTraces + return factoryOptionFunc(func(o *factoryImpl, cfgType component.Type) { + o.TracesStabilityFunc = sl.Self + o.CreateTracesFunc = CreateTracesFunc(typeChecked[consumer.Traces, Traces](creator[consumer.Traces, Traces](createTraces), cfgType)) }) } // WithMetrics overrides the default "error not supported" implementation for Factory.CreateMetrics and the default "undefined" stability level. func WithMetrics(createMetrics CreateMetricsFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factory) { - o.metricsStabilityLevel = sl - o.createMetricsFunc = createMetrics + return factoryOptionFunc(func(o *factoryImpl, cfgType component.Type) { + o.MetricsStabilityFunc = sl.Self + o.CreateMetricsFunc = CreateMetricsFunc(typeChecked[consumer.Metrics, Metrics](creator[consumer.Metrics, Metrics](createMetrics), cfgType)) }) } // WithLogs overrides the default "error not supported" implementation for Factory.CreateLogs and the default "undefined" stability level. func WithLogs(createLogs CreateLogsFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factory) { - o.logsStabilityLevel = sl - o.createLogsFunc = createLogs + return factoryOptionFunc(func(o *factoryImpl, cfgType component.Type) { + o.LogsStabilityFunc = sl.Self + o.CreateLogsFunc = CreateLogsFunc(typeChecked[consumer.Logs, Logs](creator[consumer.Logs, Logs](createLogs), cfgType)) }) } // NewFactory returns a Factory. func NewFactory(cfgType component.Type, createDefaultConfig component.CreateDefaultConfigFunc, options ...FactoryOption) Factory { - f := &factory{ - cfgType: cfgType, - CreateDefaultConfigFunc: createDefaultConfig, + f := factoryImpl{ + Factory: component.NewFactoryImpl(cfgType.Self, createDefaultConfig), } for _, opt := range options { - opt.applyOption(f) + opt.applyOption(&f, cfgType) } return f } + +func NewFactoryImpl( + factory component.Factory, + tracesFunc CreateTracesFunc, + tracesStab TracesStabilityFunc, + metricsFunc CreateMetricsFunc, + mtricsStab MetricsStabilityFunc, + createLogs CreateLogsFunc, + logsStab LogsStabilityFunc, +) Factory { + return factoryImpl{ + Factory: factory, + CreateTracesFunc: tracesFunc, + TracesStabilityFunc: tracesStab, + CreateMetricsFunc: metricsFunc, + MetricsStabilityFunc: mtricsStab, + CreateLogsFunc: createLogs, + LogsStabilityFunc: logsStab, + } +} diff --git a/receiver/xreceiver/profiles.go b/receiver/xreceiver/profiles.go index 8a912119ab2..0b0b8faf6c2 100644 --- a/receiver/xreceiver/profiles.go +++ b/receiver/xreceiver/profiles.go @@ -41,79 +41,108 @@ type Factory interface { // CreateProfilesFunc is the equivalent of Factory.CreateProfiles. type CreateProfilesFunc func(context.Context, receiver.Settings, component.Config, xconsumer.Profiles) (Profiles, error) +// ProfilesStabilityFunc is a functional way to construct Factory implementations. +type ProfilesStabilityFunc func() component.StabilityLevel + +func (f ProfilesStabilityFunc) ProfilesStability() component.StabilityLevel { + if f == nil { + return component.StabilityLevelUndefined + } + return f() +} + // FactoryOption apply changes to Factory. type FactoryOption interface { // applyOption applies the option. - applyOption(o *factoryOpts) + applyOption(o *factoryOpts, cfgType component.Type) } // factoryOptionFunc is a FactoryOption created through a function. -type factoryOptionFunc func(*factoryOpts) +type factoryOptionFunc func(*factoryOpts, component.Type) -func (f factoryOptionFunc) applyOption(o *factoryOpts) { - f(o) +func (f factoryOptionFunc) applyOption(o *factoryOpts, cfgType component.Type) { + f(o, cfgType) } -type factory struct { +type factoryImpl struct { receiver.Factory - createProfilesFunc CreateProfilesFunc - profilesStabilityLevel component.StabilityLevel + CreateProfilesFunc + ProfilesStabilityFunc } -func (f *factory) ProfilesStability() component.StabilityLevel { - return f.profilesStabilityLevel -} +var _ Factory = factoryImpl{} -func (f *factory) CreateProfiles(ctx context.Context, set receiver.Settings, cfg component.Config, next xconsumer.Profiles) (Profiles, error) { - if f.createProfilesFunc == nil { +func (f CreateProfilesFunc) CreateProfiles(ctx context.Context, set receiver.Settings, cfg component.Config, next xconsumer.Profiles) (Profiles, error) { + if f == nil { return nil, pipeline.ErrSignalNotSupported } - if set.ID.Type() != f.Type() { - return nil, internal.ErrIDMismatch(set.ID, f.Type()) - } - return f.createProfilesFunc(ctx, set, cfg, next) + return f(ctx, set, cfg, next) } type factoryOpts struct { opts []receiver.FactoryOption - *factory + factoryImpl +} + +type creator[A, B any] func(ctx context.Context, set receiver.Settings, cfg component.Config, next A) (B, error) + +func typeChecked[A, B any](cf creator[A, B], cfgType component.Type) creator[A, B] { + return func(ctx context.Context, set receiver.Settings, cfg component.Config, next A) (B, error) { + if set.ID.Type() != cfgType { + var zero B + return zero, internal.ErrIDMismatch(set.ID, cfgType) + } + return cf(ctx, set, cfg, next) + } } // WithTraces overrides the default "error not supported" implementation for Factory.CreateTraces and the default "undefined" stability level. func WithTraces(createTraces receiver.CreateTracesFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factoryOpts) { + return factoryOptionFunc(func(o *factoryOpts, cfgType component.Type) { o.opts = append(o.opts, receiver.WithTraces(createTraces, sl)) }) } // WithMetrics overrides the default "error not supported" implementation for Factory.CreateMetrics and the default "undefined" stability level. func WithMetrics(createMetrics receiver.CreateMetricsFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factoryOpts) { + return factoryOptionFunc(func(o *factoryOpts, cfgType component.Type) { o.opts = append(o.opts, receiver.WithMetrics(createMetrics, sl)) }) } // WithLogs overrides the default "error not supported" implementation for Factory.CreateLogs and the default "undefined" stability level. func WithLogs(createLogs receiver.CreateLogsFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factoryOpts) { + return factoryOptionFunc(func(o *factoryOpts, cfgType component.Type) { o.opts = append(o.opts, receiver.WithLogs(createLogs, sl)) }) } // WithProfiles overrides the default "error not supported" implementation for Factory.CreateProfiles and the default "undefined" stability level. func WithProfiles(createProfiles CreateProfilesFunc, sl component.StabilityLevel) FactoryOption { - return factoryOptionFunc(func(o *factoryOpts) { - o.profilesStabilityLevel = sl - o.createProfilesFunc = createProfiles + return factoryOptionFunc(func(o *factoryOpts, cfgType component.Type) { + o.ProfilesStabilityFunc = sl.Self + o.CreateProfilesFunc = CreateProfilesFunc(typeChecked[xconsumer.Profiles, Profiles](creator[xconsumer.Profiles, Profiles](createProfiles), cfgType)) }) } // NewFactory returns a Factory. func NewFactory(cfgType component.Type, createDefaultConfig component.CreateDefaultConfigFunc, options ...FactoryOption) Factory { - opts := factoryOpts{factory: &factory{}} + var opts factoryOpts for _, opt := range options { - opt.applyOption(&opts) + opt.applyOption(&opts, cfgType) } opts.Factory = receiver.NewFactory(cfgType, createDefaultConfig, opts.opts...) - return opts.factory + return opts.factoryImpl +} + +func NewFactoryImpl( + factory receiver.Factory, + profilesFunc CreateProfilesFunc, + profilesStab ProfilesStabilityFunc, +) Factory { + return factoryImpl{ + Factory: factory, + CreateProfilesFunc: profilesFunc, + ProfilesStabilityFunc: profilesStab, + } } diff --git a/service/go.mod b/service/go.mod index 76f06c43cbb..d6717f70a23 100644 --- a/service/go.mod +++ b/service/go.mod @@ -110,7 +110,9 @@ require ( go.opentelemetry.io/collector/config/configtls v1.34.0 // indirect go.opentelemetry.io/collector/consumer/consumererror v0.128.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.34.0 // indirect - go.opentelemetry.io/collector/extension/extensionmiddleware v0.128.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.128.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/contrib/zpages v0.61.0 // indirect @@ -202,8 +204,6 @@ replace go.opentelemetry.io/collector/config/configcompression => ../config/conf replace go.opentelemetry.io/collector/pdata/pprofile => ../pdata/pprofile -replace go.opentelemetry.io/collector/consumer/xconsumer => ../consumer/xconsumer - replace go.opentelemetry.io/collector/consumer/consumertest => ../consumer/consumertest replace go.opentelemetry.io/collector/client => ../client @@ -244,4 +244,8 @@ replace go.opentelemetry.io/collector/config/configmiddleware => ../config/confi replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/consumer/xconsumer => ../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../pdata/xpdata diff --git a/service/hostcapabilities/go.mod b/service/hostcapabilities/go.mod index d04c32f8d5b..180451e3f97 100644 --- a/service/hostcapabilities/go.mod +++ b/service/hostcapabilities/go.mod @@ -96,4 +96,6 @@ replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/co replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/pdata/xpdata => ../../pdata/xpdata