From 1d520ea4e15b7e85aa59b38d08977b64f69e76f0 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 10 Jul 2025 11:40:48 +0200 Subject: [PATCH 001/173] chore: add rfc --- rfc/cosmo-streams-v1.md | 365 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 rfc/cosmo-streams-v1.md diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md new file mode 100644 index 0000000000..ab6a9706d4 --- /dev/null +++ b/rfc/cosmo-streams-v1.md @@ -0,0 +1,365 @@ +# RFC Cosmo Streams V1 +All should use the custom modules + +## Authorization + +### Prerequisites +* In the authorization hook, we need to make the decision if the client/user is authorized at all to subscribe +* second, we have to decide which topics the user is allowed to subscribe to. + +### Additional prerequisites +We can use a similar hook also for non-stream subscriptions, to satisfy the following requirements: +* Should also allow for customers to add additional logic on JWT, like verify expiration, sign/secrets +* Should allow to return a Unauthenticated/Unauthorized message and close the subscriptions +(requested by united talent) + +## Init Func +### Prerequisites +* From the client request, derive an initial payload to resolve the first event. Can be optional. Can be implemented but might not return an initial payload. + +## Map from broker to entity +### Prerequisites +* Some customers, like Procore, have existing Kafka infra that doesn't align with their GraphQL Schema. They might use headers or other Kafka specific features. This function will take events from any broker and allow us to map them to valid entity objects. + +## Filter entity events +### Prerequisites +* Based on client request, client args, authentication, etc. the function can filter the stream for each subscriber. + +# Proposal + +## Core Types + +core.OperationContext +- add `Variables() *astjson.Value` + +core.SubscriptionContext +- `RequestContext() core.RequestContext` +- `SendError(err error)` +- `Close()` + +core.StreamNatsConfiguration +- `Subjects() []string` + +core.StreamKafkaConfiguration +- `Topics() []string` + +core.StreamRedisConfiguration +- `Channels() []string` + +core.StreamType +- `Nats` +- `Kafka` +- `Redis` + +core.StreamProvider +- `Id() string` +- `Type() core.StreamType` + +core.StreamConfiguration +- `Provider() core.StreamProvider` +- `Type() core.StreamType` +- `Nats() *core.StreamNatsConfiguration` +- `Kafka() *core.StreamKafkaConfiguration` +- `Redis() *core.StreamRedisConfiguration` + +core.StreamNatsEvent +- `Subject() string` +- `Data() []byte` + +core.StreamKafkaEvent +- `Topic() string` +- `Data() []byte` +- `Headers() map[string]string` +- `SetHeader(key string, value string)`, add a header to the event + +core.StreamRedisEvent +- `Channel() string` +- `Data() []byte` + +core.StreamEvent +- `Type() core.StreamType` +- `Redis() *core.StreamRedisEvent` +- `Kafka() *core.StreamKafkaEvent` +- `Nats() *core.StreamNatsEvent` +- `Metadata() map[string]string`, metadata that can be used to store additional information about the event and passed between hooks +- `SetMetadata(key string, value string)`, set a metadata entry +- `ToSkip() bool`, if true, the event will not be sent to the client +- `SetToSkip(toSkip bool)`, set the toSkip flag +- `Data() []byte`, get the data of the event +- `SetData(data []byte)`, set the data of the event +- `Error() error`, get the error of the event +- `SetError(err error)`, set the error of the event + +core.StreamEventWithError +- `Event() core.StreamEvent` +- `Error() error`, get the error of the event +- `SetError(err error)`, set the error of the event + +core.StreamContext +- `Configuration() *core.StreamConfiguration` +- `Metadata() map[string]string`, metadata that can be used to store additional information about the stream and passed between hooks +- `SetMetadata(key string, value string)` +- `WriteEvent(event core.StreamEvent)`, write an event to the stream +- `Close()`, close the stream + +## Hooks + +core.RouterOnSubscriptionStartHandler +- `RouterOnSubscriptionStart(ctx core.SubscriptionContext)`, called once at subscription start + - can send an error to the client with `ctx.SendError(fmt.Errorf("my custom error: %w", err))` + - can close the subscription with `ctx.Close()` + +core.RouterOnStreamStartHandler +- `RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext)`, called once at cosmo stream start, right after the subscription start hook + - can send an error to the client with `subCtx.SendError(fmt.Errorf("my custom error: %w", err))` + - can close the subscription with `subCtx.Close()` + +core.RouterOnStreamEventReceivedHandler +- `RouterOnStreamEventReceived(streamCtx core.StreamContext, event *core.StreamEventWithError)`, called once for each event as has been received from the adapter before it is delivered to the clients + - can set an error that will be delivered to the clients with `event.SetError(fmt.Errorf("my custom error: %w", err))` + - can set the toSkip flag to skip delivering the event to the clients + - can change the event data before delivering it to the clients + +core.RouterOnStreamEventToClientHandler +- `RouterOnStreamEventToClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEventWithError)`, applied before the message is delivereted to the client, executed one time for each unique combination of event and client + - can set an error that will be delivered to the client + - can set the toSkip flag to skip delivering the event to the client + - can change the event data even using client informations, before delivering it to the client + - if this hook is implemented, the event is copied before delivering it to the client, to avoid side effects between clients; this behaviour will make this hook less performant than `RouterOnStreamEventReceivedHandler`, that should be preferred if possible + +core.RouterOnStreamEventToSendHandler +- `RouterOnStreamEventToSend(streamCtx core.StreamContext, event *core.StreamEvent)`, called once for each event that is going to be sent to the adapter + - can set the toSkip flag to skip delivering the event to the adapter + - can change the event data before sending it to the adapter + +core.RouterOnStreamEventToSendWithClientHandler +- `RouterOnStreamEventToSendWithClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEvent)`, called once for each event that is going to be sent to the adapter + - can set the toSkip flag to skip delivering the event to the adapter + - can change the event data before sending it to the adapter + - can change the event data even using client informations, before delivering it to the adapter + - if this hook is implemented, the event is copied before delivering it to the hook, to avoid side effects between clients; this behaviour will make this hook less performant than `RouterOnStreamEventToSendHandler`, that should be preferred if possible + + +## Examples + +### Check if the client is allowed to subscribe to the stream + +```go +type MyModule struct { + Logger *zap.Logger +} + +func (m *MyModule) Provision(ctx *core.ModuleContext) error { + m.Logger = ctx.Logger + return nil +} + +func customCheckIfClientIsAllowedToSubscribeToStream(subCtx core.SubscriptionContext, streamCtx core.StreamContext) bool { + providerId := streamCtx.Configuration().Provider().Id() + clientScopes := subCtx.RequestContext().Authentication().Scopes() + + if slices.Contains(clientScopes, "admin") { + return true + } + + if providerId == "sharable-data" && streamCtx.Configuration().Provider().Type() == core.StreamTypeNats { + return true + } + + if providerId == "almost-sharable-data" + && streamCtx.Configuration().Provider().Type() == core.StreamTypeNats + && streamCtx.Configuration().Provider().Nats().Subjects() == []string{"public"} { + return true + } + + return false +} + +func (m *MyModule) RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext) { + if !customCheckIfClientIsAllowedToSubscribeToStream(subCtx, streamCtx) { + subCtx.SendError(fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!")) + subCtx.Close() + } +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} +``` + +### Derive the initial payload + +```go +type MyModule struct { + Logger *zap.Logger +} + +func (m *MyModule) Provision(ctx *core.ModuleContext) error { + m.Logger = ctx.Logger + return nil +} + +func (m *MyModule) RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext) { + opName := subCtx.RequestContext().Operation().Name() + opVarId := subCtx.RequestContext().Operation().Variables().GetInt("id") + if opName == "employeeSub" && opVarId == 100 { + streamCtx.WriteEvent(core.StreamEvent{ + Data: []byte(fmt.Sprintf("{\"id\": \"%d\", \"__typename\": \"Employee\"}", opVarId)), + }) + } +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} +``` + + +### Rewrite the event from a stream for all the subscription's clients + +```go +type MyModule struct { + Logger *zap.Logger +} + +func (m *MyModule) Provision(ctx *core.ModuleContext) error { + m.Logger = ctx.Logger + return nil +} + +func (m *MyModule) RouterOnStreamEventReceived(streamCtx core.StreamContext, event *core.StreamEventWithError) { + if streamCtx.Configuration().Type() != core.StreamTypeKafka { + return + } + if event.Kafka().Topic() == "topic-with-internal-data-format" { + idHeader := event.Kafka().Headers()["id"] + if idHeader == "" { + event.SetToSkip(true) + m.Logger.Warn("id is empty, skipping") + return + } + event.SetData([]byte(fmt.Sprintf("{\"id\": \"%s\", \"__typename\": \"Employee\"}", idHeader))) + } +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} +``` + + + +### Rewrite the mutation event for a mutation to copy the id in a header + +```go +type MyModule struct { + Logger *zap.Logger +} + +func (m *MyModule) Provision(ctx *core.ModuleContext) error { + m.Logger = ctx.Logger + return nil +} + +func (m *MyModule) RouterOnStreamEventToSend(streamCtx core.StreamContext, event *core.StreamEvent) { + if streamCtx.Configuration().Type() != core.StreamTypeKafka { + return + } + if event.Kafka().Topic() == "topic-with-internal-data-format" { + var data struct { + Id string `json:"id"` + } + json.Unmarshal(event.Data(), &data) + event.SetHeader("entity-id", data.Id) + } +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} +``` + + +### Filter events based on the client's scopes and the stream's configuration + +```go +type MyModule struct { + Logger *zap.Logger +} + +func (m *MyModule) Provision(ctx *core.ModuleContext) error { + m.Logger = ctx.Logger + return nil +} + +func (m *MyModule) RouterOnStreamEventToClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEventWithError) { + clientAllowedEntitiesIds, found := subCtx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + if !found { + m.Logger.Debug("allowedEntitiesIds not found, skipping") + return + } + if streamCtx.Configuration().Type() != core.StreamTypeKafka { + return + } + if event.Kafka().Topic() == "topic-with-internal-data-format" { + idHeader := event.Kafka().Headers()["id"] + if idHeader == "" { + event.SetToSkip(true) + m.Logger.Warn("id is empty, skipping") + return + } + if !slices.Contains(clientAllowedEntitiesIds, idHeader) { + event.SetToSkip(true) + m.Logger.Warn("id is not allowed, skipping") + return + } + } +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} +``` + + +# Implementation details + +The implementation of this solution will only require changes in the cosmo repo, without any changes to the engine. +This implementation will require additional changes to the hooks structures each time a new provider is added. + +# Here be dragons + +- all the hooks could be called in parallel, so we need to be careful with that +- all the hooks implementations could raise a panic, so we need to be careful with that also +- especially the `RouterOnStreamEventToClient` hook, that could be called for each client, could slow down the delivery of the event to the client and use a lot of memory +- probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks \ No newline at end of file From 290aa5fd3c875234e704b8e2e2f490a184cfadc2 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 10 Jul 2025 21:52:50 +0200 Subject: [PATCH 002/173] Add some more description --- rfc/cosmo-streams-v1.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index ab6a9706d4..9639f1d58a 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -1,33 +1,44 @@ # RFC Cosmo Streams V1 -All should use the custom modules -## Authorization +We received feedback from customers that they need a way to customize the Streams behaviour. The most important things to customize are: +- authorization checks when starting subscriptions +- send a first message to the client when a subscription is started +- map data to align with internal specifications +- filter events with custom logic -### Prerequisites +All these customization pieces should use the custom modules system. + +## Requirements + +### Authorization + +#### Prerequisites * In the authorization hook, we need to make the decision if the client/user is authorized at all to subscribe * second, we have to decide which topics the user is allowed to subscribe to. -### Additional prerequisites +#### Additional prerequisites We can use a similar hook also for non-stream subscriptions, to satisfy the following requirements: * Should also allow for customers to add additional logic on JWT, like verify expiration, sign/secrets * Should allow to return a Unauthenticated/Unauthorized message and close the subscriptions (requested by united talent) -## Init Func -### Prerequisites +### Init Func +#### Prerequisites * From the client request, derive an initial payload to resolve the first event. Can be optional. Can be implemented but might not return an initial payload. -## Map from broker to entity -### Prerequisites +### Map from broker to entity +#### Prerequisites * Some customers, like Procore, have existing Kafka infra that doesn't align with their GraphQL Schema. They might use headers or other Kafka specific features. This function will take events from any broker and allow us to map them to valid entity objects. -## Filter entity events -### Prerequisites +### Filter entity events +#### Prerequisites * Based on client request, client args, authentication, etc. the function can filter the stream for each subscriber. -# Proposal +## Proposal + +The best way to satisfy all the requirements is to add some hook in the stream lifecycle. We will also need some more data structure to allow the new hooks to read and write in the streams. -## Core Types +### Core Types to add core.OperationContext - add `Variables() *astjson.Value` @@ -102,7 +113,7 @@ core.StreamContext - `WriteEvent(event core.StreamEvent)`, write an event to the stream - `Close()`, close the stream -## Hooks +### Hooks to add core.RouterOnSubscriptionStartHandler - `RouterOnSubscriptionStart(ctx core.SubscriptionContext)`, called once at subscription start @@ -362,4 +373,4 @@ This implementation will require additional changes to the hooks structures each - all the hooks could be called in parallel, so we need to be careful with that - all the hooks implementations could raise a panic, so we need to be careful with that also - especially the `RouterOnStreamEventToClient` hook, that could be called for each client, could slow down the delivery of the event to the client and use a lot of memory -- probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks \ No newline at end of file +- probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks From da3f6a19d5efa5b4908e1de2546db9adf1f587cd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 13:34:49 +0200 Subject: [PATCH 003/173] feat: improved RFC base on feedback received --- rfc/cosmo-streams-v1.md | 675 ++++++++++++++++++++++++++-------------- 1 file changed, 441 insertions(+), 234 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 9639f1d58a..695437da19 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -1,196 +1,92 @@ # RFC Cosmo Streams V1 -We received feedback from customers that they need a way to customize the Streams behaviour. The most important things to customize are: -- authorization checks when starting subscriptions -- send a first message to the client when a subscription is started -- map data to align with internal specifications -- filter events with custom logic - -All these customization pieces should use the custom modules system. - -## Requirements - -### Authorization - -#### Prerequisites -* In the authorization hook, we need to make the decision if the client/user is authorized at all to subscribe -* second, we have to decide which topics the user is allowed to subscribe to. - -#### Additional prerequisites -We can use a similar hook also for non-stream subscriptions, to satisfy the following requirements: -* Should also allow for customers to add additional logic on JWT, like verify expiration, sign/secrets -* Should allow to return a Unauthenticated/Unauthorized message and close the subscriptions -(requested by united talent) - -### Init Func -#### Prerequisites -* From the client request, derive an initial payload to resolve the first event. Can be optional. Can be implemented but might not return an initial payload. - -### Map from broker to entity -#### Prerequisites -* Some customers, like Procore, have existing Kafka infra that doesn't align with their GraphQL Schema. They might use headers or other Kafka specific features. This function will take events from any broker and allow us to map them to valid entity objects. - -### Filter entity events -#### Prerequisites -* Based on client request, client args, authentication, etc. the function can filter the stream for each subscriber. - -## Proposal - -The best way to satisfy all the requirements is to add some hook in the stream lifecycle. We will also need some more data structure to allow the new hooks to read and write in the streams. - -### Core Types to add - -core.OperationContext -- add `Variables() *astjson.Value` - -core.SubscriptionContext -- `RequestContext() core.RequestContext` -- `SendError(err error)` -- `Close()` - -core.StreamNatsConfiguration -- `Subjects() []string` - -core.StreamKafkaConfiguration -- `Topics() []string` - -core.StreamRedisConfiguration -- `Channels() []string` - -core.StreamType -- `Nats` -- `Kafka` -- `Redis` - -core.StreamProvider -- `Id() string` -- `Type() core.StreamType` - -core.StreamConfiguration -- `Provider() core.StreamProvider` -- `Type() core.StreamType` -- `Nats() *core.StreamNatsConfiguration` -- `Kafka() *core.StreamKafkaConfiguration` -- `Redis() *core.StreamRedisConfiguration` - -core.StreamNatsEvent -- `Subject() string` -- `Data() []byte` - -core.StreamKafkaEvent -- `Topic() string` -- `Data() []byte` -- `Headers() map[string]string` -- `SetHeader(key string, value string)`, add a header to the event - -core.StreamRedisEvent -- `Channel() string` -- `Data() []byte` - -core.StreamEvent -- `Type() core.StreamType` -- `Redis() *core.StreamRedisEvent` -- `Kafka() *core.StreamKafkaEvent` -- `Nats() *core.StreamNatsEvent` -- `Metadata() map[string]string`, metadata that can be used to store additional information about the event and passed between hooks -- `SetMetadata(key string, value string)`, set a metadata entry -- `ToSkip() bool`, if true, the event will not be sent to the client -- `SetToSkip(toSkip bool)`, set the toSkip flag -- `Data() []byte`, get the data of the event -- `SetData(data []byte)`, set the data of the event -- `Error() error`, get the error of the event -- `SetError(err error)`, set the error of the event - -core.StreamEventWithError -- `Event() core.StreamEvent` -- `Error() error`, get the error of the event -- `SetError(err error)`, set the error of the event - -core.StreamContext -- `Configuration() *core.StreamConfiguration` -- `Metadata() map[string]string`, metadata that can be used to store additional information about the stream and passed between hooks -- `SetMetadata(key string, value string)` -- `WriteEvent(event core.StreamEvent)`, write an event to the stream -- `Close()`, close the stream - -### Hooks to add - -core.RouterOnSubscriptionStartHandler -- `RouterOnSubscriptionStart(ctx core.SubscriptionContext)`, called once at subscription start - - can send an error to the client with `ctx.SendError(fmt.Errorf("my custom error: %w", err))` - - can close the subscription with `ctx.Close()` - -core.RouterOnStreamStartHandler -- `RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext)`, called once at cosmo stream start, right after the subscription start hook - - can send an error to the client with `subCtx.SendError(fmt.Errorf("my custom error: %w", err))` - - can close the subscription with `subCtx.Close()` - -core.RouterOnStreamEventReceivedHandler -- `RouterOnStreamEventReceived(streamCtx core.StreamContext, event *core.StreamEventWithError)`, called once for each event as has been received from the adapter before it is delivered to the clients - - can set an error that will be delivered to the clients with `event.SetError(fmt.Errorf("my custom error: %w", err))` - - can set the toSkip flag to skip delivering the event to the clients - - can change the event data before delivering it to the clients - -core.RouterOnStreamEventToClientHandler -- `RouterOnStreamEventToClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEventWithError)`, applied before the message is delivereted to the client, executed one time for each unique combination of event and client - - can set an error that will be delivered to the client - - can set the toSkip flag to skip delivering the event to the client - - can change the event data even using client informations, before delivering it to the client - - if this hook is implemented, the event is copied before delivering it to the client, to avoid side effects between clients; this behaviour will make this hook less performant than `RouterOnStreamEventReceivedHandler`, that should be preferred if possible - -core.RouterOnStreamEventToSendHandler -- `RouterOnStreamEventToSend(streamCtx core.StreamContext, event *core.StreamEvent)`, called once for each event that is going to be sent to the adapter - - can set the toSkip flag to skip delivering the event to the adapter - - can change the event data before sending it to the adapter - -core.RouterOnStreamEventToSendWithClientHandler -- `RouterOnStreamEventToSendWithClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEvent)`, called once for each event that is going to be sent to the adapter - - can set the toSkip flag to skip delivering the event to the adapter - - can change the event data before sending it to the adapter - - can change the event data even using client informations, before delivering it to the adapter - - if this hook is implemented, the event is copied before delivering it to the hook, to avoid side effects between clients; this behaviour will make this hook less performant than `RouterOnStreamEventToSendHandler`, that should be preferred if possible - - -## Examples - -### Check if the client is allowed to subscribe to the stream +Based on customer feedback, we've identified the need for more customizable stream behavior. The key areas for customization include: +- Authorization: implementing authorization checks at the start of subscriptions +- Initial message: sending an initial message to clients upon subscription start +- Data mapping: mapping data to align with internal specifications +- Event filtering: filtering events using custom logic + +Let's explore how we can address each of these requirements. + +## Authorization +To support authorization, we need a hook that enables two key decisions: +- Whether the client or user is authorized to initiate the subscription at all +- Which topics the client is permitted to subscribe to + +Additionally, a similar mechanism is required for non-stream subscriptions, allowing: +- Custom JWT validation logic (e.g., expiration checks, signature verification, secret handling) +- The ability to reject unauthenticated or unauthorized requests and close the subscription accordingly + +We already allow some customization using RouterOnRequestHandler, but it has no access to the stream data. To get them, we need to add a new hook that will be called right before the subscription is started. + +### Example: check if the client is allowed to subscribe to the stream ```go -type MyModule struct { - Logger *zap.Logger +// the structs are reported only with the fields that are used in the example +type StreamContext interface { + ProviderType() string + ProviderId() string + // the subscription configuration is specific for each provider + SubscriptionConfiguration() []byte } -func (m *MyModule) Provision(ctx *core.ModuleContext) error { - m.Logger = ctx.Logger - return nil +type RequestContext interface { + Authentication() *core.Authentication +} + +type SubscriptionContext interface { + RequestContext() RequestContext + StreamContext() StreamContext } -func customCheckIfClientIsAllowedToSubscribeToStream(subCtx core.SubscriptionContext, streamCtx core.StreamContext) bool { - providerId := streamCtx.Configuration().Provider().Id() - clientScopes := subCtx.RequestContext().Authentication().Scopes() +// This is the new hook that will be called once at stream start +type SubscriptionOnStartHandler interface { + SubscriptionOnStart(ctx SubscriptionContext) error +} + +// already defined in the provider package +type SubscriptionEventConfiguration struct { + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` +} + +type MyModule struct {} + +func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionContext) bool { + providerType := ctx.StreamContext().ProviderType() + providerId := ctx.StreamContext().ProviderId() + clientScopes := ctx.RequestContext().Authentication().Scopes() if slices.Contains(clientScopes, "admin") { return true } - if providerId == "sharable-data" && streamCtx.Configuration().Provider().Type() == core.StreamTypeNats { + if providerId == "sharable-data" && providerType == "nats" { return true } + + // unmarshal the subscription data, specific for each provider + var subscriptionConfiguration nats.SubscriptionEventConfiguration + err := json.Unmarshal(streamCtx.SubscriptionConfiguration(), &subscriptionConfiguration) + if err != nil { + return false + } if providerId == "almost-sharable-data" - && streamCtx.Configuration().Provider().Type() == core.StreamTypeNats - && streamCtx.Configuration().Provider().Nats().Subjects() == []string{"public"} { + && providerType == "nats" + && subscriptionConfiguration.Subjects == []string{"public"} { return true } return false } -func (m *MyModule) RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext) { - if !customCheckIfClientIsAllowedToSubscribeToStream(subCtx, streamCtx) { - subCtx.SendError(fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!")) - subCtx.Close() +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { + if !customCheckIfClientIsAllowedToSubscribe(ctx) { + return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") } + return nil } func (m *MyModule) Module() core.ModuleInfo { @@ -204,26 +100,99 @@ func (m *MyModule) Module() core.ModuleInfo { } ``` -### Derive the initial payload +### Proposal + +Add a new hook to the subscription lifecycle, `SubscriptionOnStart`, that will be called once at subscription start. + +The arguments of the hook are: +* `ctx SubscriptionContext`: the subscription context, that contains the request context and, optionally, the stream context + +RequestContext already exists and need no changes, but SubscriptionContext is new. + +The hook should return an error if the client is not allowed to subscribe to the stream, and the subscription will not be started. +The hook should return nil if the client is allowed to subscribe to the stream, and the subscription will be started. + +I evaluated the possibility to just add the SubscriptionContext to the request context and use it inside one of the existing hooks, +but it would be hard to build the subscription context without executing the pubsub code. + +The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as is used by the provider. This will allow the hooks system to be agnostic of the provider type, so that adding a new provider will not require any changes to the hooks system. + +## Initial message +When starting a subscription, the client will send a query to the server. +The query contains the operation name and the variables. +And then the client will have to wait for the server to send the initial message. +This waiting could lead to a bad user experience, because the client can't see anything until the initial message is received. +To solve this, we can emit an initial message on subscription start. + +To emit an initial message on subscription start, we need access to the stream context (to get the provider type and id) and also the query that the client sent. +The variables are really important to know to allow the module to use them to emit the initial message. +E.g. if someone start a subscription with employee id 100, the custom module can emit the initial message with that id inside. + +### Example ```go -type MyModule struct { - Logger *zap.Logger +// the structs are reported only with the fields that are used in the example +type StreamEvent interface { + Data() []byte + SetData(data []byte) } -func (m *MyModule) Provision(ctx *core.ModuleContext) error { - m.Logger = ctx.Logger - return nil +type StreamContext interface { + ProviderType() string + WriteEvent(event core.StreamEvent) +} + +type OperationContext interface { + Name() string + Variables() *astjson.Value +} + +type RequestContext interface { + Operation() core.OperationContext } -func (m *MyModule) RouterOnStreamStart(subCtx core.SubscriptionContext, streamCtx core.StreamContext) { - opName := subCtx.RequestContext().Operation().Name() - opVarId := subCtx.RequestContext().Operation().Variables().GetInt("id") - if opName == "employeeSub" && opVarId == 100 { - streamCtx.WriteEvent(core.StreamEvent{ +type SubscriptionContext struct { + RequestContext() RequestContext + StreamContext() StreamContext +} + +// This is the new hook that will be called once at stream start +type SubscriptionOnStartHandler interface { + SubscriptionOnStart(ctx SubscriptionContext) error +} + +// already defined in the provider package, but we need to add the metadata field +type PublishAndRequestEventConfiguration struct { + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` +} + +type MyModule struct {} + +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { + opName := ctx.RequestContext().Operation().Name() + opVarId := ctx.RequestContext().Operation().Variables().GetInt("id") + if opName == "employeeSub" { + publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ + ProviderID: "employee-stream", + Subject: "employee-stream", Data: []byte(fmt.Sprintf("{\"id\": \"%d\", \"__typename\": \"Employee\"}", opVarId)), - }) + Metadata: map[string]string{ + "entity-id": fmt.Sprintf("%d", opVarId), + }, + } + data, err := json.Marshal(publishAndRequestEventConfiguration) + if err != nil { + return fmt.Errorf("error marshalling data: %w", err) + } + + // create the event with the data and the provider type + evt := core.NewStreamEvent(ctx.StreamContext().ProviderType(), data) + ctx.StreamContext().WriteEvent(evt) } + return nil } func (m *MyModule) Module() core.ModuleInfo { @@ -237,32 +206,92 @@ func (m *MyModule) Module() core.ModuleInfo { } ``` +### Proposal + +Using the new `SubscriptionOnStart`hook, that we already introduced to solve the previous requirement, we can emit the initial message on subscription start. +We will also need access to operation variables, that right now are not available in the request context. + +To emit the message I propose to add a new method to the stream context, `WriteEvent`, that will emit the event to the stream at the lowest level. +The message will go through all the hooks, so that it will be just like any other event received from the provider. + +The `StreamEvent` contains the data as is used by the provider. This will allow the hooks system to be agnostic of the provider type, so that adding a new provider will not require any changes to the hooks system. + +Emitting the initial message with this hook will guarantee that the client will receive the message and it will receive it before the first event from the provider is received. + +## Data mapping -### Rewrite the event from a stream for all the subscription's clients +The current way we have to emit and read the data from the stream is not flexible enough. +We need to be able to map the data from an external format to the internal format, and also to map the data from the internal format to an external format. +### Example 1, rewrite the event received from the provider to a format that is usable from cosmo streams ```go -type MyModule struct { - Logger *zap.Logger +// the structs are reported only with the fields that are used in the example +type StreamEvent interface { + Data() []byte + SetData(data []byte) } -func (m *MyModule) Provision(ctx *core.ModuleContext) error { - m.Logger = ctx.Logger - return nil +// This is the new hook that will be called once for each event received from the provider +type StreamOnEventReceivedHandler interface { + StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error +} + +// already defined in the provider package, but we need to add the metadata field +type PublishAndRequestEventConfiguration struct { + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` } -func (m *MyModule) RouterOnStreamEventReceived(streamCtx core.StreamContext, event *core.StreamEventWithError) { - if streamCtx.Configuration().Type() != core.StreamTypeKafka { - return +// to be defined in the provider package +type ReceivedEventConfiguration struct { + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` +} + +type MyModule struct {} + +func (m *MyModule) StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error { + var receivedEventConfiguration nats.ReceivedEventConfiguration + + // unmarshal the event data that we received from the provider + err := json.Unmarshal(event.Data(), &receivedEventConfiguration) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) } - if event.Kafka().Topic() == "topic-with-internal-data-format" { - idHeader := event.Kafka().Headers()["id"] - if idHeader == "" { - event.SetToSkip(true) - m.Logger.Warn("id is empty, skipping") - return + + // prepare the event to send with all the changes that we want to do to the data + if receivedEventConfiguration.Subject == "topic-with-internal-data-format" { + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` + } + err := json.Unmarshal(event.Data(), &dataReceived) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) + } + var dataForStream struct { + Id string `json:"id"` + Name string `json:"__typename"` } - event.SetData([]byte(fmt.Sprintf("{\"id\": \"%s\", \"__typename\": \"Employee\"}", idHeader))) + dataForStream.Id = dataReceived.EmployeeId + dataForStream.Name = "Employee" + + publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ + ProviderID: receivedEventConfiguration.ProviderID, + Subject: receivedEventConfiguration.Subject, + Data: dataForStream, + Metadata: receivedEventConfiguration.Metadata, + } + data, err := json.Marshal(publishAndRequestEventConfiguration) + if err != nil { + return fmt.Errorf("error marshalling data: %w", err) + } + event.SetData(data) } + return nil } func (m *MyModule) Module() core.ModuleInfo { @@ -276,31 +305,93 @@ func (m *MyModule) Module() core.ModuleInfo { } ``` +### Example 2, rewrite the event before emitting it to the provider to a format that is usable from external systems +```go +// the structs are reported only with the fields that are used in the example +type StreamEvent interface { + Data() []byte + SetData(data []byte) +} +type StreamContext interface { + WriteEvent(event core.StreamEvent) +} -### Rewrite the mutation event for a mutation to copy the id in a header +// This is the new hook that will be called once for each event that is going to be sent to the provider +type StreamOnEventToSendHandler interface { + StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error +} -```go -type MyModule struct { - Logger *zap.Logger +// already defined in the provider package +type SubscriptionEventConfiguration struct { + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` } -func (m *MyModule) Provision(ctx *core.ModuleContext) error { - m.Logger = ctx.Logger - return nil +// already defined in the provider package, but we need to add the metadata field +type PublishAndRequestEventConfiguration struct { + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` } -func (m *MyModule) RouterOnStreamEventToSend(streamCtx core.StreamContext, event *core.StreamEvent) { - if streamCtx.Configuration().Type() != core.StreamTypeKafka { - return +type MyModule struct {} + +func (m *MyModule) StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error { + // unmarshal the subscription data, specific for each provider + var subscriptionConfiguration nats.SubscriptionEventConfiguration + err := json.Unmarshal(streamCtx.SubscriptionConfiguration(), &subscriptionConfiguration) + if err != nil { + return false } - if event.Kafka().Topic() == "topic-with-internal-data-format" { + + if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { + // unmarshal the event data that we received from the provider + var oldEventConfiguration nats.PublishAndRequestEventConfiguration + err := json.Unmarshal(event.Data(), &oldEventConfiguration) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) + } + + // unmarshal the data of the message the we are expecting var data struct { Id string `json:"id"` + Name string `json:"__typename"` + } + err := json.Unmarshal(oldEventConfiguration.Data, &data) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) + } + + // prepare the data to send to the provider to be usable from external systems + var dataToSend struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + dataToSend.EmployeeId = data.Id + dataToSend.OtherField = "Custom value" + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return fmt.Errorf("error marshalling data: %w", err) + } + publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ + ProviderID: oldEventConfiguration.ProviderID, + Subject: oldEventConfiguration.Subject, + Data: dataToSendMarshalled, + Metadata: map[string]string{ + "entity-id": data.Id, + "entity-domain": "employee", + }, + } + eventData, err := json.Marshal(publishAndRequestEventConfiguration) + if err != nil { + return fmt.Errorf("error marshalling data: %w", err) } - json.Unmarshal(event.Data(), &data) - event.SetHeader("entity-id", data.Id) + event.SetData(eventData) } + return nil } func (m *MyModule) Module() core.ModuleInfo { @@ -314,41 +405,120 @@ func (m *MyModule) Module() core.ModuleInfo { } ``` +### Proposal -### Filter events based on the client's scopes and the stream's configuration +Add two new hooks to the stream lifecycle, `StreamOnEventReceived` and `StreamOnEventToSend`, that will be called once for each event received from the provider and once for each event that is going to be sent to the provider. + +The `StreamOnEventReceived` hook will be called once for each event received from the provider, so that it will be possible to rewrite the event data to a format usable inside cosmo streams. +The `StreamOnEventToSend` hook will be called once for each event that is going to be sent to the provider, so that it will be possible to rewrite the event data to a format usable from external systems. + +The arguments of the hooks are: +* `ctx StreamContext`: the stream context, that contains the id and type of the stream +* `event core.StreamEvent`: the event received from the provider or the event that is going to be sent to the provider + +The hook should return an error if the event cannot be processed, and the event will not be processed. +The hook should return nil if the event can be processed, and the event will be processed. + +I also thought about exposing the subscription context to the hooks, but it would be too easy to misuse it and use some data specific to the subscription and add it to an event that will not be sent only to that provider. To make it safe I should copy the whole event data for each pair of event and subscription that needs to receive it. + +This proposal requires the introduction of a new format of events that the pubsub system uses. +As an example, for NATS we are currently using the `PublishAndRequestEventConfiguration` struct, when writing events, but when we are reading events, we only pass down the data of the event. We have to build an intermediate struct that will allow us to access metadata, data and other fields of the event. In the example 1 we are using the `ReceivedEventConfiguration` struct for this purpose. + +This change is sensible but it would be needed anyway to support metadata in the events. + +#### Do we need two new hooks? + +Another possibile solution for mapping the outward data would be to use the already existing middleware hooks `RouterOnRequestHandler` or the `RouterMiddlewareHandler` to "eat" the mutation and access to the stream context and emit the event to the stream. But this would require exposing a stream context on the request lifecycle, that is difficult. Also this will require some coordination to be sure that an event emitted on the stream is sent only after the subscription is started. +Also, this solution is not usable on the subscription side of the streams: +- the middleware hooks are linked to the request lifecycle, so it would be hard to use them to rewrite the event data; +- when we are going to use the streams feature internally, we will still need to provide a way to rewrite the event data, so we will need to add a new hook to the subscription lifecycle; + +So I think that the best solution is to add two new hooks to the stream lifecycle. + + +## Event filtering + +We need to allow customers to filter events base on custom logic. We actually only provide declarative filters, and they are really limited. + +### Example, filter events based on streams configuration and client's scopes ```go -type MyModule struct { - Logger *zap.Logger +// the structs are reported only with the fields that are used in the example +type StreamEvent interface { + Data() []byte } -func (m *MyModule) Provision(ctx *core.ModuleContext) error { - m.Logger = ctx.Logger - return nil +type StreamContext interface { + WriteEvent(event core.StreamEvent) } -func (m *MyModule) RouterOnStreamEventToClient(subCtx core.SubscriptionContext, streamCtx core.StreamContext, event *core.StreamEventWithError) { - clientAllowedEntitiesIds, found := subCtx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] +type OperationContext interface { + Name() string + Variables() *astjson.Value +} + +type RequestContext interface { + Authentication() *core.Authentication + Operation() core.OperationContext +} + +type SubscriptionContext struct { + RequestContext() RequestContext + StreamContext() StreamContext +} + +// This is the new hook that will be called before delivering an event to the client +type StreamOnEventFilterHandler interface { + // return true to skip the event, false to deliver it + StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool +} + +// already defined in the provider package +type SubscriptionEventConfiguration struct { + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` +} + +// to be defined in the provider package +type ReceivedEventConfiguration struct { + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` +} + +type MyModule struct {} + +func (m *MyModule) StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool { + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { - m.Logger.Debug("allowedEntitiesIds not found, skipping") - return + return true } - if streamCtx.Configuration().Type() != core.StreamTypeKafka { - return + + var subscriptionConfiguration nats.SubscriptionEventConfiguration + err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) } - if event.Kafka().Topic() == "topic-with-internal-data-format" { - idHeader := event.Kafka().Headers()["id"] - if idHeader == "" { - event.SetToSkip(true) - m.Logger.Warn("id is empty, skipping") - return + + if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { + var receivedEventConfiguration nats.ReceivedEventConfiguration + err := json.Unmarshal(event.Data(), &receivedEventConfiguration) + if err != nil { + return fmt.Errorf("error unmarshalling data: %w", err) + } + + idHeader, ok := receivedEventConfiguration.Metadata["entity-id"] + if !ok { + return true } - if !slices.Contains(clientAllowedEntitiesIds, idHeader) { - event.SetToSkip(true) - m.Logger.Warn("id is not allowed, skipping") - return + if slices.Contains(clientAllowedEntitiesIds, idHeader) { + // the event is delivered to the client only if the id is in the allowed entities ids + return false } } + return true } func (m *MyModule) Module() core.ModuleInfo { @@ -360,8 +530,45 @@ func (m *MyModule) Module() core.ModuleInfo { }, } } -``` +### Proposal + +Add a new hook to the stream lifecycle, `StreamOnEventFilter`, that will be called before delivering an event to the client. + +The arguments of the hook are: +* `ctx core.SubscriptionContext`: the subscription context, that contains the request context and, optionally, the stream context +* `event core.StreamEvent`: the event received from the provider or the event that is going to be sent to the provider + +The hook should return true to skip the event, false to deliver it. + +Ideally we could use the StreamOnEventReceivedHandler to filter the events, but it would require to add the subscription context to the stream context, that is not a good idea: it would be easy to misuse it and use some data specific to the subscription and add it to an event that will not be sent only to that client. Also, the StreamOnEventReceivedHandler is called for each event received from the provider, and this new hook should be called for each combination of event and subscription that is going to be delivered to the client. + +## Architecture + +With this proposal, we are going to add some hooks to the subscription lifecycle, and some hooks to the stream lifecycle. + +### Subscription lifecycle +Start subscription + │ + └─▶ core.SubscriptionOnStartHandler (Early return, Custom Authentication Logic) + │ + └─▶ "Subscription started" + +### Stream lifecycle + +An event is received from the provider + │ + └─▶ core.StreamOnEventReceivedHandler (Data mapping) + │ + └─▶ core.StreamOnEventFilterHandler (Filtering) + │ + └─▶ "Deliver event to client" + +A mutation is sent from the client + │ + └─▶ core.StreamOnEventToSendHandler (Data mapping) + │ + └─▶ "Send event to provider" # Implementation details @@ -372,5 +579,5 @@ This implementation will require additional changes to the hooks structures each - all the hooks could be called in parallel, so we need to be careful with that - all the hooks implementations could raise a panic, so we need to be careful with that also -- especially the `RouterOnStreamEventToClient` hook, that could be called for each client, could slow down the delivery of the event to the client and use a lot of memory +- in the hook `StreamOnEventFilter` a user could change the event data without considering that the changes could be sent to other clients also. - probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks From 82fed9f4eebc43988796db97bb359d39e9a6fcb2 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 13:39:12 +0200 Subject: [PATCH 004/173] chore: clean a bit structures --- rfc/cosmo-streams-v1.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 695437da19..a9bea18816 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -449,7 +449,7 @@ type StreamEvent interface { } type StreamContext interface { - WriteEvent(event core.StreamEvent) + SubscriptionConfiguration() []byte } type OperationContext interface { @@ -459,7 +459,6 @@ type OperationContext interface { type RequestContext interface { Authentication() *core.Authentication - Operation() core.OperationContext } type SubscriptionContext struct { From 12527901d1b7d6db55298aa87d404ea8a2a5674c Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 14:25:21 +0200 Subject: [PATCH 005/173] chore: small fixes --- rfc/cosmo-streams-v1.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index a9bea18816..2d40d47c66 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -68,14 +68,14 @@ func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionContext) bool { // unmarshal the subscription data, specific for each provider var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(streamCtx.SubscriptionConfiguration(), &subscriptionConfiguration) + err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) if err != nil { return false } - if providerId == "almost-sharable-data" - && providerType == "nats" - && subscriptionConfiguration.Subjects == []string{"public"} { + if providerId == "almost-sharable-data" && + providerType == "nats" && + slices.Equal(subscriptionConfiguration.Subjects, []string{"public"}) { return true } @@ -342,7 +342,7 @@ type MyModule struct {} func (m *MyModule) StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error { // unmarshal the subscription data, specific for each provider var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(streamCtx.SubscriptionConfiguration(), &subscriptionConfiguration) + err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) if err != nil { return false } @@ -498,14 +498,14 @@ func (m *MyModule) StreamOnEventFilter(ctx core.SubscriptionContext, event core. var subscriptionConfiguration nats.SubscriptionEventConfiguration err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) + return true } if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { var receivedEventConfiguration nats.ReceivedEventConfiguration err := json.Unmarshal(event.Data(), &receivedEventConfiguration) if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) + return true } idHeader, ok := receivedEventConfiguration.Metadata["entity-id"] From 00080254d920883f7e5fb6cf0ec09212a87caf96 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 14:32:13 +0200 Subject: [PATCH 006/173] chore: fix indentations --- rfc/cosmo-streams-v1.md | 137 ++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 2d40d47c66..3bebf66f61 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -41,14 +41,14 @@ type SubscriptionContext interface { // This is the new hook that will be called once at stream start type SubscriptionOnStartHandler interface { - SubscriptionOnStart(ctx SubscriptionContext) error + SubscriptionOnStart(ctx SubscriptionContext) error } // already defined in the provider package type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Subjects []string `json:"subjects"` - StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` } type MyModule struct {} @@ -83,20 +83,20 @@ func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionContext) bool { } func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { - if !customCheckIfClientIsAllowedToSubscribe(ctx) { - return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") - } - return nil + if !customCheckIfClientIsAllowedToSubscribe(ctx) { + return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") + } + return nil } func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } } ``` @@ -158,14 +158,14 @@ type SubscriptionContext struct { // This is the new hook that will be called once at stream start type SubscriptionOnStartHandler interface { - SubscriptionOnStart(ctx SubscriptionContext) error + SubscriptionOnStart(ctx SubscriptionContext) error } // already defined in the provider package, but we need to add the metadata field type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` } @@ -196,13 +196,13 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { } func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } } ``` @@ -233,22 +233,22 @@ type StreamEvent interface { // This is the new hook that will be called once for each event received from the provider type StreamOnEventReceivedHandler interface { - StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error + StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error } // already defined in the provider package, but we need to add the metadata field type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` } // to be defined in the provider package type ReceivedEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` } @@ -295,13 +295,13 @@ func (m *MyModule) StreamOnEventReceived(ctx StreamContext, event core.StreamEve } func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } } ``` @@ -319,21 +319,21 @@ type StreamContext interface { // This is the new hook that will be called once for each event that is going to be sent to the provider type StreamOnEventToSendHandler interface { - StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error + StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error } // already defined in the provider package type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Subjects []string `json:"subjects"` - StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` } // already defined in the provider package, but we need to add the metadata field type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` } @@ -395,13 +395,13 @@ func (m *MyModule) StreamOnEventToSend(ctx StreamContext, event core.StreamEvent } func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } } ``` @@ -469,21 +469,21 @@ type SubscriptionContext struct { // This is the new hook that will be called before delivering an event to the client type StreamOnEventFilterHandler interface { // return true to skip the event, false to deliver it - StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool + StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool } // already defined in the provider package type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Subjects []string `json:"subjects"` - StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` + ProviderID string `json:"providerId"` + Subjects []string `json:"subjects"` + StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` } // to be defined in the provider package type ReceivedEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID string `json:"providerId"` + Subject string `json:"subject"` + Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` } @@ -521,14 +521,15 @@ func (m *MyModule) StreamOnEventFilter(ctx core.SubscriptionContext, event core. } func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } } +``` ### Proposal From 26bb7db95d27180e3edae713d52fbfc7a3302221 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 14:49:13 +0200 Subject: [PATCH 007/173] chore: small fixes --- rfc/cosmo-streams-v1.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 3bebf66f61..01d29ff051 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -127,7 +127,7 @@ To solve this, we can emit an initial message on subscription start. To emit an initial message on subscription start, we need access to the stream context (to get the provider type and id) and also the query that the client sent. The variables are really important to know to allow the module to use them to emit the initial message. -E.g. if someone start a subscription with employee id 100, the custom module can emit the initial message with that id inside. +E.g. if someone start a subscription with variable employee id 100, the custom module can emit the initial message with that id inside. ### Example ```go @@ -144,6 +144,7 @@ type StreamContext interface { type OperationContext interface { Name() string + // the variables are currently not available, so we need to add them here Variables() *astjson.Value } @@ -208,7 +209,7 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Using the new `SubscriptionOnStart`hook, that we already introduced to solve the previous requirement, we can emit the initial message on subscription start. +Using the new `SubscriptionOnStart` hook, that we already introduced to solve the previous requirement, we can emit the initial message on subscription start. We will also need access to operation variables, that right now are not available in the request context. To emit the message I propose to add a new method to the stream context, `WriteEvent`, that will emit the event to the stream at the lowest level. @@ -570,14 +571,18 @@ A mutation is sent from the client │ └─▶ "Send event to provider" +### Data flow + +We will have to change the format of the event data that is sent inside the router: today we are using directly the data that will be sent to the provider, but we will need to add a structure where we can add additional fields (metadata, etc.) to the event. + # Implementation details The implementation of this solution will only require changes in the cosmo repo, without any changes to the engine. -This implementation will require additional changes to the hooks structures each time a new provider is added. +This implementation will not require additional changes to the hooks structures each time a new provider is added. # Here be dragons - all the hooks could be called in parallel, so we need to be careful with that - all the hooks implementations could raise a panic, so we need to be careful with that also -- in the hook `StreamOnEventFilter` a user could change the event data without considering that the changes could be sent to other clients also. +- in the hook `StreamOnEventFilter` a user could change the event data without considering that the changes could be sent to other clients also, so we need to advise the users to be careful with this hook - probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks From d909dc0b58293d3c304b345b8d327569df54093a Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 17:09:16 +0200 Subject: [PATCH 008/173] chore: improved english flow --- rfc/cosmo-streams-v1.md | 142 +++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 73 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 01d29ff051..d4b4f24c36 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -1,25 +1,26 @@ # RFC Cosmo Streams V1 Based on customer feedback, we've identified the need for more customizable stream behavior. The key areas for customization include: -- Authorization: implementing authorization checks at the start of subscriptions -- Initial message: sending an initial message to clients upon subscription start -- Data mapping: mapping data to align with internal specifications -- Event filtering: filtering events using custom logic +- **Authorization**: Implementing authorization checks at the start of subscriptions +- **Initial message**: Sending an initial message to clients upon subscription start +- **Data mapping**: Transforming data to align with internal specifications +- **Event filtering**: Filtering events using custom logic Let's explore how we can address each of these requirements. ## Authorization + To support authorization, we need a hook that enables two key decisions: -- Whether the client or user is authorized to initiate the subscription at all +- Whether the client or user is authorized to initiate the subscription - Which topics the client is permitted to subscribe to Additionally, a similar mechanism is required for non-stream subscriptions, allowing: - Custom JWT validation logic (e.g., expiration checks, signature verification, secret handling) - The ability to reject unauthenticated or unauthorized requests and close the subscription accordingly -We already allow some customization using RouterOnRequestHandler, but it has no access to the stream data. To get them, we need to add a new hook that will be called right before the subscription is started. +We already allow some customization using `RouterOnRequestHandler`, but it has no access to the stream data. To access this data, we need to add a new hook that will be called immediately before the subscription starts. -### Example: check if the client is allowed to subscribe to the stream +### Example: Check if the client is allowed to subscribe to the stream ```go // the structs are reported only with the fields that are used in the example @@ -104,32 +105,26 @@ func (m *MyModule) Module() core.ModuleInfo { Add a new hook to the subscription lifecycle, `SubscriptionOnStart`, that will be called once at subscription start. -The arguments of the hook are: -* `ctx SubscriptionContext`: the subscription context, that contains the request context and, optionally, the stream context +The hook arguments are: +* `ctx SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context -RequestContext already exists and need no changes, but SubscriptionContext is new. +`RequestContext` already exists and requires no changes, but `SubscriptionContext` is new. -The hook should return an error if the client is not allowed to subscribe to the stream, and the subscription will not be started. -The hook should return nil if the client is allowed to subscribe to the stream, and the subscription will be started. +The hook should return an error if the client is not allowed to subscribe to the stream, preventing the subscription from starting. +The hook should return `nil` if the client is allowed to subscribe to the stream, allowing the subscription to proceed. -I evaluated the possibility to just add the SubscriptionContext to the request context and use it inside one of the existing hooks, -but it would be hard to build the subscription context without executing the pubsub code. +I evaluated the possibility of adding the `SubscriptionContext` to the request context and using it within one of the existing hooks, but it would be difficult to build the subscription context without executing the pubsub code. -The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as is used by the provider. This will allow the hooks system to be agnostic of the provider type, so that adding a new provider will not require any changes to the hooks system. +The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. -## Initial message +## Initial Message -When starting a subscription, the client will send a query to the server. -The query contains the operation name and the variables. -And then the client will have to wait for the server to send the initial message. -This waiting could lead to a bad user experience, because the client can't see anything until the initial message is received. -To solve this, we can emit an initial message on subscription start. +When starting a subscription, the client sends a query to the server containing the operation name and variables. The client must then wait for the broker to send the initial message. This waiting period can lead to a poor user experience, as the client cannot display anything until the initial message is received. To address this, we can emit an initial message on subscription start. -To emit an initial message on subscription start, we need access to the stream context (to get the provider type and id) and also the query that the client sent. -The variables are really important to know to allow the module to use them to emit the initial message. -E.g. if someone start a subscription with variable employee id 100, the custom module can emit the initial message with that id inside. +To emit an initial message on subscription start, we need access to the stream context (to get the provider type and ID) and the query that the client sent. The variables are particularly important, as they allow the module to use them in the initial message. For example, if someone starts a subscription with employee ID 100 as a variable, the custom module can include that ID in the initial message. ### Example + ```go // the structs are reported only with the fields that are used in the example type StreamEvent interface { @@ -209,22 +204,20 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Using the new `SubscriptionOnStart` hook, that we already introduced to solve the previous requirement, we can emit the initial message on subscription start. -We will also need access to operation variables, that right now are not available in the request context. +Using the new `SubscriptionOnStart` hook that we introduced for the previous requirement, we can emit the initial message on subscription start. We will also need access to operation variables, which are currently not available in the request context. -To emit the message I propose to add a new method to the stream context, `WriteEvent`, that will emit the event to the stream at the lowest level. -The message will go through all the hooks, so that it will be just like any other event received from the provider. +To emit the message, I propose adding a new method to the stream context, `WriteEvent`, which will emit the event to the stream at the lowest level. The message will pass through all hooks, making it behave like any other event received from the provider. -The `StreamEvent` contains the data as is used by the provider. This will allow the hooks system to be agnostic of the provider type, so that adding a new provider will not require any changes to the hooks system. +The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. -Emitting the initial message with this hook will guarantee that the client will receive the message and it will receive it before the first event from the provider is received. +Emitting the initial message with this hook ensures that the client will receive the message before the first event from the provider is received. -## Data mapping +## Data Mapping -The current way we have to emit and read the data from the stream is not flexible enough. -We need to be able to map the data from an external format to the internal format, and also to map the data from the internal format to an external format. +The current approach for emitting and reading data from the stream is not flexible enough. We need to be able to map data from an external format to the internal format, and vice versa. + +### Example 1: Rewrite the event received from the provider to a format that is usable by Cosmo streams -### Example 1, rewrite the event received from the provider to a format that is usable from cosmo streams ```go // the structs are reported only with the fields that are used in the example type StreamEvent interface { @@ -306,7 +299,8 @@ func (m *MyModule) Module() core.ModuleInfo { } ``` -### Example 2, rewrite the event before emitting it to the provider to a format that is usable from external systems +### Example 2: Rewrite the event before emitting it to the provider to a format that is usable by external systems + ```go // the structs are reported only with the fields that are used in the example type StreamEvent interface { @@ -408,40 +402,39 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Add two new hooks to the stream lifecycle, `StreamOnEventReceived` and `StreamOnEventToSend`, that will be called once for each event received from the provider and once for each event that is going to be sent to the provider. +Add two new hooks to the stream lifecycle: `StreamOnEventReceived` and `StreamOnEventToSend`, which will be called once for each event received from the provider and once for each event that is going to be sent to the provider. -The `StreamOnEventReceived` hook will be called once for each event received from the provider, so that it will be possible to rewrite the event data to a format usable inside cosmo streams. -The `StreamOnEventToSend` hook will be called once for each event that is going to be sent to the provider, so that it will be possible to rewrite the event data to a format usable from external systems. +The `StreamOnEventReceived` hook will be called for each event received from the provider, making it possible to rewrite the event data to a format usable within Cosmo streams. +The `StreamOnEventToSend` hook will be called for each event that is going to be sent to the provider, making it possible to rewrite the event data to a format usable by external systems. -The arguments of the hooks are: -* `ctx StreamContext`: the stream context, that contains the id and type of the stream -* `event core.StreamEvent`: the event received from the provider or the event that is going to be sent to the provider +The hook arguments are: +* `ctx StreamContext`: The stream context, which contains the ID and type of the stream +* `event core.StreamEvent`: The event received from the provider or the event that is going to be sent to the provider -The hook should return an error if the event cannot be processed, and the event will not be processed. -The hook should return nil if the event can be processed, and the event will be processed. +The hook should return an error if the event cannot be processed, preventing the event from being processed. +The hook should return `nil` if the event can be processed, allowing the event to proceed. -I also thought about exposing the subscription context to the hooks, but it would be too easy to misuse it and use some data specific to the subscription and add it to an event that will not be sent only to that provider. To make it safe I should copy the whole event data for each pair of event and subscription that needs to receive it. +I also considered exposing the subscription context to the hooks, but this would be too easy to misuse. Users might add subscription-specific data to an event that will be sent to multiple providers. To make it safe, I would need to copy the entire event data for each pair of event and subscription that needs to receive it. -This proposal requires the introduction of a new format of events that the pubsub system uses. -As an example, for NATS we are currently using the `PublishAndRequestEventConfiguration` struct, when writing events, but when we are reading events, we only pass down the data of the event. We have to build an intermediate struct that will allow us to access metadata, data and other fields of the event. In the example 1 we are using the `ReceivedEventConfiguration` struct for this purpose. +This proposal requires introducing a new format for events that the pubsub system uses. For example, with NATS we currently use the `PublishAndRequestEventConfiguration` struct when writing events, but when reading events, we only pass down the event data. We need to build an intermediate struct that allows us to access metadata, data, and other event fields. In Example 1, we use the `ReceivedEventConfiguration` struct for this purpose. -This change is sensible but it would be needed anyway to support metadata in the events. +This change is significant but would be needed anyway to support metadata in events. #### Do we need two new hooks? -Another possibile solution for mapping the outward data would be to use the already existing middleware hooks `RouterOnRequestHandler` or the `RouterMiddlewareHandler` to "eat" the mutation and access to the stream context and emit the event to the stream. But this would require exposing a stream context on the request lifecycle, that is difficult. Also this will require some coordination to be sure that an event emitted on the stream is sent only after the subscription is started. -Also, this solution is not usable on the subscription side of the streams: -- the middleware hooks are linked to the request lifecycle, so it would be hard to use them to rewrite the event data; -- when we are going to use the streams feature internally, we will still need to provide a way to rewrite the event data, so we will need to add a new hook to the subscription lifecycle; +Another possible solution for mapping outward data would be to use the existing middleware hooks `RouterOnRequestHandler` or `RouterMiddlewareHandler` to intercept the mutation, access the stream context, and emit the event to the stream. However, this would require exposing a stream context in the request lifecycle, which is difficult. It would also require coordination to ensure that an event emitted on the stream is sent only after the subscription starts. -So I think that the best solution is to add two new hooks to the stream lifecycle. +Additionally, this solution is not usable on the subscription side of streams: +- The middleware hooks are linked to the request lifecycle, making it difficult to use them to rewrite event data +- When we use the streams feature internally, we will still need to provide a way to rewrite event data, requiring a new hook in the subscription lifecycle +Therefore, I believe the best solution is to add two new hooks to the stream lifecycle. -## Event filtering +## Event Filtering -We need to allow customers to filter events base on custom logic. We actually only provide declarative filters, and they are really limited. +We need to allow customers to filter events based on custom logic. We currently only provide declarative filters, which are quite limited. -### Example, filter events based on streams configuration and client's scopes +### Example: Filter events based on stream configuration and client's scopes ```go // the structs are reported only with the fields that are used in the example @@ -536,27 +529,30 @@ func (m *MyModule) Module() core.ModuleInfo { Add a new hook to the stream lifecycle, `StreamOnEventFilter`, that will be called before delivering an event to the client. -The arguments of the hook are: -* `ctx core.SubscriptionContext`: the subscription context, that contains the request context and, optionally, the stream context -* `event core.StreamEvent`: the event received from the provider or the event that is going to be sent to the provider +The hook arguments are: +* `ctx core.SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context +* `event core.StreamEvent`: The event received from the provider or the event that is going to be sent to the provider -The hook should return true to skip the event, false to deliver it. +The hook should return `true` to skip the event, `false` to deliver it. -Ideally we could use the StreamOnEventReceivedHandler to filter the events, but it would require to add the subscription context to the stream context, that is not a good idea: it would be easy to misuse it and use some data specific to the subscription and add it to an event that will not be sent only to that client. Also, the StreamOnEventReceivedHandler is called for each event received from the provider, and this new hook should be called for each combination of event and subscription that is going to be delivered to the client. +Ideally, we could use the `StreamOnEventReceivedHandler` to filter events, but this would require adding the subscription context to the stream context, which is not a good idea. It would be easy to misuse by adding subscription-specific data to an event that should not be sent only to that client. Also, the `StreamOnEventReceivedHandler` is called for each event received from the provider, while this new hook should be called for each combination of event and subscription that is going to be delivered to the client. ## Architecture -With this proposal, we are going to add some hooks to the subscription lifecycle, and some hooks to the stream lifecycle. +With this proposal, we will add hooks to both the subscription lifecycle and the stream lifecycle. -### Subscription lifecycle +### Subscription Lifecycle +``` Start subscription │ └─▶ core.SubscriptionOnStartHandler (Early return, Custom Authentication Logic) │ └─▶ "Subscription started" +``` -### Stream lifecycle +### Stream Lifecycle +``` An event is received from the provider │ └─▶ core.StreamOnEventReceivedHandler (Data mapping) @@ -570,19 +566,19 @@ A mutation is sent from the client └─▶ core.StreamOnEventToSendHandler (Data mapping) │ └─▶ "Send event to provider" +``` -### Data flow +### Data Flow -We will have to change the format of the event data that is sent inside the router: today we are using directly the data that will be sent to the provider, but we will need to add a structure where we can add additional fields (metadata, etc.) to the event. +We will need to change the format of the event data sent within the router. Today we use the data that will be sent to the provider directly, but we will need to add a structure where we can include additional fields (metadata, etc.) in the event. -# Implementation details +## Implementation Details -The implementation of this solution will only require changes in the cosmo repo, without any changes to the engine. -This implementation will not require additional changes to the hooks structures each time a new provider is added. +The implementation of this solution will only require changes in the Cosmo repository, without any changes to the engine. This implementation will not require additional changes to the hooks structures each time a new provider is added. -# Here be dragons +## Considerations and Risks -- all the hooks could be called in parallel, so we need to be careful with that -- all the hooks implementations could raise a panic, so we need to be careful with that also -- in the hook `StreamOnEventFilter` a user could change the event data without considering that the changes could be sent to other clients also, so we need to advise the users to be careful with this hook -- probably we should also add metrics to track how much time is spent in each hook, to help customers pinpoint slow hooks +- All hooks could be called in parallel, so we need to handle concurrency carefully +- All hook implementations could raise a panic, so we need to implement proper error handling +- In the `StreamOnEventFilter` hook, a user could change the event data without considering that the changes could be sent to other clients as well, so we need to advise users to be careful with this hook +- We should add metrics to track how much time is spent in each hook, to help customers identify slow hooks From d032b6718f1d21def79eb876f025322fdb2d393d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 17:22:30 +0200 Subject: [PATCH 009/173] chore: fix wrong logic in an example --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index d4b4f24c36..e8c4cd3f0f 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -262,7 +262,7 @@ func (m *MyModule) StreamOnEventReceived(ctx StreamContext, event core.StreamEve var dataReceived struct { EmployeeId string `json:"EmployeeId"` } - err := json.Unmarshal(event.Data(), &dataReceived) + err := json.Unmarshal(receivedEventConfiguration.Data, &dataReceived) if err != nil { return fmt.Errorf("error unmarshalling data: %w", err) } From 9c69cdc8a3a76d25354e4a1c9ed3b131ad177069 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 17:32:20 +0200 Subject: [PATCH 010/173] chore: change StreamOnEventFilter meaning of the returned boolean --- rfc/cosmo-streams-v1.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index e8c4cd3f0f..816b1aac89 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -462,7 +462,7 @@ type SubscriptionContext struct { // This is the new hook that will be called before delivering an event to the client type StreamOnEventFilterHandler interface { - // return true to skip the event, false to deliver it + // return false to skip the event, true to deliver it StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool } @@ -486,32 +486,32 @@ type MyModule struct {} func (m *MyModule) StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool { clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { - return true + return false } var subscriptionConfiguration nats.SubscriptionEventConfiguration err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) if err != nil { - return true + return false } if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { var receivedEventConfiguration nats.ReceivedEventConfiguration err := json.Unmarshal(event.Data(), &receivedEventConfiguration) if err != nil { - return true + return false } idHeader, ok := receivedEventConfiguration.Metadata["entity-id"] if !ok { - return true + return false } if slices.Contains(clientAllowedEntitiesIds, idHeader) { // the event is delivered to the client only if the id is in the allowed entities ids - return false + return true } } - return true + return false } func (m *MyModule) Module() core.ModuleInfo { @@ -533,7 +533,7 @@ The hook arguments are: * `ctx core.SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context * `event core.StreamEvent`: The event received from the provider or the event that is going to be sent to the provider -The hook should return `true` to skip the event, `false` to deliver it. +The hook should return `false` to skip the event, `true` to deliver it. Ideally, we could use the `StreamOnEventReceivedHandler` to filter events, but this would require adding the subscription context to the stream context, which is not a good idea. It would be easy to misuse by adding subscription-specific data to an event that should not be sent only to that client. Also, the `StreamOnEventReceivedHandler` is called for each event received from the provider, while this new hook should be called for each combination of event and subscription that is going to be delivered to the client. From 80f7097f5944c686d16f537df7f6256c01e7b227 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 11 Jul 2025 18:14:16 +0200 Subject: [PATCH 011/173] chore: small fixes to example --- rfc/cosmo-streams-v1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 816b1aac89..33dc353b00 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -337,12 +337,12 @@ type MyModule struct {} func (m *MyModule) StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error { // unmarshal the subscription data, specific for each provider var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) + err := json.Unmarshal(ctx.SubscriptionConfiguration(), &subscriptionConfiguration) if err != nil { - return false + return err } - if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { + if slices.Contains(subscriptionConfiguration.Subjects, "topic-with-internal-data-format") { // unmarshal the event data that we received from the provider var oldEventConfiguration nats.PublishAndRequestEventConfiguration err := json.Unmarshal(event.Data(), &oldEventConfiguration) From 7969ec9430be2de0cd44fbec89b3a9d528680a09 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sat, 12 Jul 2025 20:45:28 +0200 Subject: [PATCH 012/173] chore: address feedback --- rfc/cosmo-streams-v1.md | 476 ++++++++++++++++++++-------------------- 1 file changed, 235 insertions(+), 241 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 33dc353b00..3c9113e29d 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -24,29 +24,31 @@ We already allow some customization using `RouterOnRequestHandler`, but it has n ```go // the structs are reported only with the fields that are used in the example +type SubscriptionEventConfiguration interface { + ProviderID() string +} + type StreamContext interface { ProviderType() string - ProviderId() string - // the subscription configuration is specific for each provider - SubscriptionConfiguration() []byte + SubscriptionConfiguration() SubscriptionEventConfiguration } type RequestContext interface { Authentication() *core.Authentication } -type SubscriptionContext interface { +type SubscriptionOnStartHookContext interface { RequestContext() RequestContext StreamContext() StreamContext } -// This is the new hook that will be called once at stream start +// This is the new hook that will be called once at subscription start type SubscriptionOnStartHandler interface { - SubscriptionOnStart(ctx SubscriptionContext) error + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } // already defined in the provider package -type SubscriptionEventConfiguration struct { +type NatsSubscriptionEventConfiguration struct { ProviderID string `json:"providerId"` Subjects []string `json:"subjects"` StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` @@ -54,36 +56,32 @@ type SubscriptionEventConfiguration struct { type MyModule struct {} -func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionContext) bool { - providerType := ctx.StreamContext().ProviderType() - providerId := ctx.StreamContext().ProviderId() +func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) bool { + cfg, ok := ctx.StreamContext().SubscriptionConfiguration().(*NatsSubscriptionEventConfiguration) + if !ok { + return true + } + + providerId := cfg.ProviderID clientScopes := ctx.RequestContext().Authentication().Scopes() if slices.Contains(clientScopes, "admin") { return true } - if providerId == "sharable-data" && providerType == "nats" { + if providerId == "sharable-data" { return true } - - // unmarshal the subscription data, specific for each provider - var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) - if err != nil { - return false - } if providerId == "almost-sharable-data" && - providerType == "nats" && - slices.Equal(subscriptionConfiguration.Subjects, []string{"public"}) { + slices.Equal(cfg.Subjects, []string{"public"}) { return true } return false } -func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { if !customCheckIfClientIsAllowedToSubscribe(ctx) { return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") } @@ -103,7 +101,7 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Add a new hook to the subscription lifecycle, `SubscriptionOnStart`, that will be called once at subscription start. +Add a new hook to the subscription lifecycle, `SubscriptionOnStartHandler`, that will be called once at subscription start. The hook arguments are: * `ctx SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context @@ -115,7 +113,7 @@ The hook should return `nil` if the client is allowed to subscribe to the stream I evaluated the possibility of adding the `SubscriptionContext` to the request context and using it within one of the existing hooks, but it would be difficult to build the subscription context without executing the pubsub code. -The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. +The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. ## Initial Message @@ -147,31 +145,31 @@ type RequestContext interface { Operation() core.OperationContext } -type SubscriptionContext struct { +type SubscriptionOnStartHookContext struct { RequestContext() RequestContext StreamContext() StreamContext } // This is the new hook that will be called once at stream start type SubscriptionOnStartHandler interface { - SubscriptionOnStart(ctx SubscriptionContext) error + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } -// already defined in the provider package, but we need to add the metadata field -type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` +// each provider will have its own event type that implements the StreamEvent interface +type NatsEvent struct { + ProviderID string + Subject string + Data json.RawMessage + Metadata map[string]string } type MyModule struct {} -func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { opName := ctx.RequestContext().Operation().Name() opVarId := ctx.RequestContext().Operation().Variables().GetInt("id") if opName == "employeeSub" { - publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ + evt := &NatsEvent{ ProviderID: "employee-stream", Subject: "employee-stream", Data: []byte(fmt.Sprintf("{\"id\": \"%d\", \"__typename\": \"Employee\"}", opVarId)), @@ -179,13 +177,6 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionContext) error { "entity-id": fmt.Sprintf("%d", opVarId), }, } - data, err := json.Marshal(publishAndRequestEventConfiguration) - if err != nil { - return fmt.Errorf("error marshalling data: %w", err) - } - - // create the event with the data and the provider type - evt := core.NewStreamEvent(ctx.StreamContext().ProviderType(), data) ctx.StreamContext().WriteEvent(evt) } return nil @@ -208,7 +199,9 @@ Using the new `SubscriptionOnStart` hook that we introduced for the previous req To emit the message, I propose adding a new method to the stream context, `WriteEvent`, which will emit the event to the stream at the lowest level. The message will pass through all hooks, making it behave like any other event received from the provider. -The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. +The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the event to the specific type for the provider. If the custom modules only need to read the data, they can use the `Data()`/`SetData()` methods without casting the event. + +This change will require adding a new type in each provider package to represent the event with additional fields (metadata, etc.). This is a significant change, but it is necessary to support additional data in events, anyway, even if we don't expose them to the custom modules. Emitting the initial message with this hook ensures that the client will receive the message before the first event from the provider is received. @@ -225,67 +218,86 @@ type StreamEvent interface { SetData(data []byte) } -// This is the new hook that will be called once for each event received from the provider -type StreamOnEventReceivedHandler interface { - StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error +type SubscriptionEventConfiguration interface { + ProviderID() string } -// already defined in the provider package, but we need to add the metadata field -type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` -} +type StreamBatchDirection string -// to be defined in the provider package -type ReceivedEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` +const ( + StreamBatchDirectionInbound StreamBatchDirection = "inbound" + StreamBatchDirectionOutbound StreamBatchDirection = "outbound" +) + +type StreamBatchEventHookContext interface { + Direction() StreamBatchDirection } -type MyModule struct {} +// each provider will have its own event type that implements the StreamEvent interface +type NatsEvent struct { + ProviderID string + Subject string + Data json.RawMessage + Metadata map[string]string +} -func (m *MyModule) StreamOnEventReceived(ctx StreamContext, event core.StreamEvent) error { - var receivedEventConfiguration nats.ReceivedEventConfiguration +// StreamBatchEventHook processes a batch of stream events (inbound or outbound). +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamBatchEventHook interface { + OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} - // unmarshal the event data that we received from the provider - err := json.Unmarshal(event.Data(), &receivedEventConfiguration) - if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) +func (m *MyModule) OnStreamEvents( + ctx StreamBatchEventHookContext, + events []StreamEvent, +) ([]StreamEvent, error) { + // we only rewrite the data for inbound events + if ctx.Direction() == StreamBatchDirectionOutbound { + return events, nil } - // prepare the event to send with all the changes that we want to do to the data - if receivedEventConfiguration.Subject == "topic-with-internal-data-format" { - var dataReceived struct { - EmployeeId string `json:"EmployeeId"` - } - err := json.Unmarshal(receivedEventConfiguration.Data, &dataReceived) - if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) - } - var dataForStream struct { - Id string `json:"id"` - Name string `json:"__typename"` + newEvents := make([]StreamEvent, 0, len(events)) + for _, evt := range events { + if natsEvent, ok := evt.(*NatsEvent); ok { + if natsEvent.Subject == "topic-with-internal-data-format" { + // rewrite the event data to a format that is usable by Cosmo streams + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + var dataForStream struct { + Id string `json:"id"` + Name string `json:"__typename"` + } + dataForStream.Id = dataReceived.EmployeeId + dataForStream.Name = "Employee" + + dataForStreamMarshalled, err := json.Marshal(dataForStream) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + newEvent := &NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataForStreamMarshalled, + Metadata: natsEvent.Metadata, + } + newEvents = append(newEvents, newEvent) + continue + } } - dataForStream.Id = dataReceived.EmployeeId - dataForStream.Name = "Employee" - - publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ - ProviderID: receivedEventConfiguration.ProviderID, - Subject: receivedEventConfiguration.Subject, - Data: dataForStream, - Metadata: receivedEventConfiguration.Metadata, - } - data, err := json.Marshal(publishAndRequestEventConfiguration) - if err != nil { - return fmt.Errorf("error marshalling data: %w", err) - } - event.SetData(data) + newEvents = append(newEvents, evt) } - return nil + + return events, nil } func (m *MyModule) Module() core.ModuleInfo { @@ -308,85 +320,88 @@ type StreamEvent interface { SetData(data []byte) } -type StreamContext interface { - WriteEvent(event core.StreamEvent) +type SubscriptionEventConfiguration interface { + ProviderID() string } -// This is the new hook that will be called once for each event that is going to be sent to the provider -type StreamOnEventToSendHandler interface { - StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error -} +type StreamBatchDirection string -// already defined in the provider package -type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Subjects []string `json:"subjects"` - StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` -} +const ( + StreamBatchDirectionInbound StreamBatchDirection = "inbound" + StreamBatchDirectionOutbound StreamBatchDirection = "outbound" +) -// already defined in the provider package, but we need to add the metadata field -type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` +type StreamBatchEventHookContext interface { + Direction() StreamBatchDirection } -type MyModule struct {} +// each provider will have its own event type that implements the StreamEvent interface +type NatsEvent struct { + ProviderID string + Subject string + Data json.RawMessage + Metadata map[string]string +} +// StreamBatchEventHook processes a batch of stream events (inbound or outbound). +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamBatchEventHook interface { + OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} -func (m *MyModule) StreamOnEventToSend(ctx StreamContext, event core.StreamEvent) error { - // unmarshal the subscription data, specific for each provider - var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(ctx.SubscriptionConfiguration(), &subscriptionConfiguration) - if err != nil { - return err +func (m *MyModule) OnStreamEvents( + ctx StreamBatchEventHookContext, + events []StreamEvent, +) ([]StreamEvent, error) { + // we only rewrite the data for outbound events + if ctx.Direction() == StreamBatchDirectionInbound { + return events, nil } - - if slices.Contains(subscriptionConfiguration.Subjects, "topic-with-internal-data-format") { - // unmarshal the event data that we received from the provider - var oldEventConfiguration nats.PublishAndRequestEventConfiguration - err := json.Unmarshal(event.Data(), &oldEventConfiguration) - if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) - } - // unmarshal the data of the message the we are expecting - var data struct { - Id string `json:"id"` - Name string `json:"__typename"` - } - err := json.Unmarshal(oldEventConfiguration.Data, &data) - if err != nil { - return fmt.Errorf("error unmarshalling data: %w", err) + newEvents := make([]StreamEvent, 0, len(events)) + for _, evt := range events { + if natsEvent, ok := evt.(*NatsEvent); ok { + if natsEvent.Subject == "topic-with-internal-data-format" { + // unmarshal the event data that we received from the provider + var dataReceived struct { + Id string `json:"id"` + TypeName string `json:"__typename"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // prepare the data to send to the provider to be usable from external systems + var dataToSend struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + dataToSend.EmployeeId = dataReceived.Id + dataToSend.OtherField = "Custom value" + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + newEvent := &NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataToSendMarshalled, + Metadata: map[string]string{ + "entity-id": dataReceived.Id, + "entity-domain": "employee", + }, + } + newEvents = append(newEvents, newEvent) + continue + } } - - // prepare the data to send to the provider to be usable from external systems - var dataToSend struct { - EmployeeId string `json:"EmployeeId"` - OtherField string `json:"OtherField"` - } - dataToSend.EmployeeId = data.Id - dataToSend.OtherField = "Custom value" - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return fmt.Errorf("error marshalling data: %w", err) - } - publishAndRequestEventConfiguration := nats.PublishAndRequestEventConfiguration{ - ProviderID: oldEventConfiguration.ProviderID, - Subject: oldEventConfiguration.Subject, - Data: dataToSendMarshalled, - Metadata: map[string]string{ - "entity-id": data.Id, - "entity-domain": "employee", - }, - } - eventData, err := json.Marshal(publishAndRequestEventConfiguration) - if err != nil { - return fmt.Errorf("error marshalling data: %w", err) - } - event.SetData(eventData) + newEvents = append(newEvents, evt) } - return nil + return newEvents, nil } func (m *MyModule) Module() core.ModuleInfo { @@ -402,33 +417,28 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Add two new hooks to the stream lifecycle: `StreamOnEventReceived` and `StreamOnEventToSend`, which will be called once for each event received from the provider and once for each event that is going to be sent to the provider. +Add a new hooks to the stream lifecycle `StreamBatchEventHook` which will be called once for each event received from the provider and once for each event that is going to be sent to the provider. -The `StreamOnEventReceived` hook will be called for each event received from the provider, making it possible to rewrite the event data to a format usable within Cosmo streams. -The `StreamOnEventToSend` hook will be called for each event that is going to be sent to the provider, making it possible to rewrite the event data to a format usable by external systems. +The `StreamBatchEventHook` will be called for each event received from the provider and each event that is going to be sent to the provider, making it possible to rewrite the event data to a format usable within Cosmo streams or by external systems. The hook arguments are: -* `ctx StreamContext`: The stream context, which contains the ID and type of the stream -* `event core.StreamEvent`: The event received from the provider or the event that is going to be sent to the provider - -The hook should return an error if the event cannot be processed, preventing the event from being processed. -The hook should return `nil` if the event can be processed, allowing the event to proceed. +* `ctx StreamBatchEventHookContext`: The stream context, which contains the ID and type of the stream (inbound or outbound) +* `events []StreamEvent`: The events received from the provider or the events that are going to be sent to the provider -I also considered exposing the subscription context to the hooks, but this would be too easy to misuse. Users might add subscription-specific data to an event that will be sent to multiple providers. To make it safe, I would need to copy the entire event data for each pair of event and subscription that needs to receive it. +The hook will return a new slice of events that will be used to emit the events to the client or to the provider. +The hook will also return an error if one of the events cannot be processed, preventing the event from being processed. -This proposal requires introducing a new format for events that the pubsub system uses. For example, with NATS we currently use the `PublishAndRequestEventConfiguration` struct when writing events, but when reading events, we only pass down the event data. We need to build an intermediate struct that allows us to access metadata, data, and other event fields. In Example 1, we use the `ReceivedEventConfiguration` struct for this purpose. +I also considered exposing the subscription context to the hook, but this would be too easy to misuse. Users might add subscription-specific data to an event that will be sent to multiple providers. To make it safe, I would need to copy the entire event data for each pair of event and subscription that needs to receive it. -This change is significant but would be needed anyway to support metadata in events. - -#### Do we need two new hooks? +#### Do we need a new hook? Another possible solution for mapping outward data would be to use the existing middleware hooks `RouterOnRequestHandler` or `RouterMiddlewareHandler` to intercept the mutation, access the stream context, and emit the event to the stream. However, this would require exposing a stream context in the request lifecycle, which is difficult. It would also require coordination to ensure that an event emitted on the stream is sent only after the subscription starts. Additionally, this solution is not usable on the subscription side of streams: -- The middleware hooks are linked to the request lifecycle, making it difficult to use them to rewrite event data +- The middleware hook is linked to the request lifecycle, making it difficult to use them to rewrite event data - When we use the streams feature internally, we will still need to provide a way to rewrite event data, requiring a new hook in the subscription lifecycle -Therefore, I believe the best solution is to add two new hooks to the stream lifecycle. +Therefore, I believe the best solution is to add a new hooks to the stream lifecycle. ## Event Filtering @@ -440,78 +450,65 @@ We need to allow customers to filter events based on custom logic. We currently // the structs are reported only with the fields that are used in the example type StreamEvent interface { Data() []byte + SetData(data []byte) } -type StreamContext interface { - SubscriptionConfiguration() []byte +type SubscriptionEventConfiguration interface { + ProviderID() string } -type OperationContext interface { - Name() string - Variables() *astjson.Value -} +type StreamBatchDirection string -type RequestContext interface { - Authentication() *core.Authentication -} +const ( + StreamBatchDirectionInbound StreamBatchDirection = "inbound" + StreamBatchDirectionOutbound StreamBatchDirection = "outbound" +) -type SubscriptionContext struct { +type StreamBatchEventHookContext interface { + Direction() StreamBatchDirection RequestContext() RequestContext - StreamContext() StreamContext } -// This is the new hook that will be called before delivering an event to the client -type StreamOnEventFilterHandler interface { - // return false to skip the event, true to deliver it - StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool +// each provider will have its own event type that implements the StreamEvent interface +type NatsEvent struct { + ProviderID string + Subject string + Data json.RawMessage + Metadata map[string]string } - -// already defined in the provider package -type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Subjects []string `json:"subjects"` - StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` -} - -// to be defined in the provider package -type ReceivedEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` +// StreamBatchEventHook processes a batch of stream events (inbound or outbound). +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamBatchEventHook interface { + OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) } type MyModule struct {} -func (m *MyModule) StreamOnEventFilter(ctx core.SubscriptionContext, event core.StreamEvent) bool { +func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) { + newEvents := make([]StreamEvent, 0, len(events)) clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { - return false + return newEvents, nil } - var subscriptionConfiguration nats.SubscriptionEventConfiguration - err := json.Unmarshal(ctx.StreamContext().SubscriptionConfiguration(), &subscriptionConfiguration) - if err != nil { - return false - } - - if subscriptionConfiguration.Subjects == []string{"topic-with-internal-data-format"} { - var receivedEventConfiguration nats.ReceivedEventConfiguration - err := json.Unmarshal(event.Data(), &receivedEventConfiguration) - if err != nil { - return false - } - - idHeader, ok := receivedEventConfiguration.Metadata["entity-id"] - if !ok { - return false - } - if slices.Contains(clientAllowedEntitiesIds, idHeader) { - // the event is delivered to the client only if the id is in the allowed entities ids - return true + for _, evt := range events { + if natsEvent, ok := evt.(*NatsEvent); ok { + if natsEvent.Subject == "topic-with-internal-data-format" { + idHeader, ok := natsEvent.Metadata["entity-id"] + if !ok { + continue + } + if slices.Contains(clientAllowedEntitiesIds, idHeader) { + newEvents = append(newEvents, evt) + } + } } } - return false + return newEvents, nil } func (m *MyModule) Module() core.ModuleInfo { @@ -527,19 +524,18 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Add a new hook to the stream lifecycle, `StreamOnEventFilter`, that will be called before delivering an event to the client. +We can use the new `StreamBatchEventHook` to filter events based on the stream configuration and the client's scopes. The hook arguments are: -* `ctx core.SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context -* `event core.StreamEvent`: The event received from the provider or the event that is going to be sent to the provider +* `ctx StreamBatchEventHookContext`: The stream context, which contains the ID and type of the stream (inbound or outbound) and the request context +* `events []StreamEvent`: The events received from the provider or the events that are going to be sent to the provider -The hook should return `false` to skip the event, `true` to deliver it. - -Ideally, we could use the `StreamOnEventReceivedHandler` to filter events, but this would require adding the subscription context to the stream context, which is not a good idea. It would be easy to misuse by adding subscription-specific data to an event that should not be sent only to that client. Also, the `StreamOnEventReceivedHandler` is called for each event received from the provider, while this new hook should be called for each combination of event and subscription that is going to be delivered to the client. +The hook will return a new slice of events that will be used to emit the events to the client or to the provider. +The hook will also return an error if one of the events cannot be processed, preventing the event from being processed. ## Architecture -With this proposal, we will add hooks to both the subscription lifecycle and the stream lifecycle. +With this proposal, we will add a new hook to the subscription and stream lifecycles. ### Subscription Lifecycle ``` @@ -553,17 +549,15 @@ Start subscription ### Stream Lifecycle ``` -An event is received from the provider - │ - └─▶ core.StreamOnEventReceivedHandler (Data mapping) +One or more batched events are received from the provider │ - └─▶ core.StreamOnEventFilterHandler (Filtering) + └─▶ core.StreamBatchEventHook (Data mapping, Filtering) │ - └─▶ "Deliver event to client" + └─▶ "Deliver events to client" -A mutation is sent from the client +One or more batched events are sent to the provider │ - └─▶ core.StreamOnEventToSendHandler (Data mapping) + └─▶ core.StreamBatchEventHook (Data mapping, Filtering) │ └─▶ "Send event to provider" ``` @@ -580,5 +574,5 @@ The implementation of this solution will only require changes in the Cosmo repos - All hooks could be called in parallel, so we need to handle concurrency carefully - All hook implementations could raise a panic, so we need to implement proper error handling -- In the `StreamOnEventFilter` hook, a user could change the event data without considering that the changes could be sent to other clients as well, so we need to advise users to be careful with this hook +- Especially the casting of the event to the specific type for the provider could raise a panic if the event is not of the expected type and the developer is not using the type check - We should add metrics to track how much time is spent in each hook, to help customers identify slow hooks From 5312f145543dc25bbb6897ac00c060391c8d19b3 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sat, 12 Jul 2025 20:52:24 +0200 Subject: [PATCH 013/173] chore: limit filter example to inbound events --- rfc/cosmo-streams-v1.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 3c9113e29d..c328353861 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -489,6 +489,11 @@ type StreamBatchEventHook interface { type MyModule struct {} func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) { + // we only filter the events for inbound events + if ctx.Direction() == StreamBatchDirectionOutbound { + return events, nil + } + newEvents := make([]StreamEvent, 0, len(events)) clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { From 259b2a06767ca8c81c3bda6ae974e029decf2c68 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 11:59:35 +0200 Subject: [PATCH 014/173] chore: separate inbound and outbound hooks, added some complete examples --- rfc/cosmo-streams-v1.md | 382 +++++++++++++++++++++++++++++++++------- 1 file changed, 317 insertions(+), 65 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index c328353861..84536e663e 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -222,15 +222,7 @@ type SubscriptionEventConfiguration interface { ProviderID() string } -type StreamBatchDirection string - -const ( - StreamBatchDirectionInbound StreamBatchDirection = "inbound" - StreamBatchDirectionOutbound StreamBatchDirection = "outbound" -) - type StreamBatchEventHookContext interface { - Direction() StreamBatchDirection } // each provider will have its own event type that implements the StreamEvent interface @@ -241,7 +233,7 @@ type NatsEvent struct { Metadata map[string]string } -// StreamBatchEventHook processes a batch of stream events (inbound or outbound). +// StreamBatchEventHook processes a batch of inbound stream events // // Return: // - empty slice: drop all events. @@ -251,15 +243,12 @@ type StreamBatchEventHook interface { OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) } +type MyModule struct {} + func (m *MyModule) OnStreamEvents( ctx StreamBatchEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { - // we only rewrite the data for inbound events - if ctx.Direction() == StreamBatchDirectionOutbound { - return events, nil - } - newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { if natsEvent, ok := evt.(*NatsEvent); ok { @@ -324,15 +313,16 @@ type SubscriptionEventConfiguration interface { ProviderID() string } -type StreamBatchDirection string - -const ( - StreamBatchDirectionInbound StreamBatchDirection = "inbound" - StreamBatchDirectionOutbound StreamBatchDirection = "outbound" -) +type StreamPublishEventHookContext interface {} -type StreamBatchEventHookContext interface { - Direction() StreamBatchDirection +// StreamPublishEventHook processes a batch of outbound stream events +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamPublishEventHook interface { + OnPublishEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) } // each provider will have its own event type that implements the StreamEvent interface @@ -342,25 +332,13 @@ type NatsEvent struct { Data json.RawMessage Metadata map[string]string } -// StreamBatchEventHook processes a batch of stream events (inbound or outbound). -// -// Return: -// - empty slice: drop all events. -// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). -// err != nil: abort the subscription with an error. -type StreamBatchEventHook interface { - OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) -} -func (m *MyModule) OnStreamEvents( - ctx StreamBatchEventHookContext, +type MyModule struct {} + +func (m *MyModule) OnPublishEvents( + ctx StreamPublishEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { - // we only rewrite the data for outbound events - if ctx.Direction() == StreamBatchDirectionInbound { - return events, nil - } - newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { if natsEvent, ok := evt.(*NatsEvent); ok { @@ -417,9 +395,9 @@ func (m *MyModule) Module() core.ModuleInfo { ### Proposal -Add a new hooks to the stream lifecycle `StreamBatchEventHook` which will be called once for each event received from the provider and once for each event that is going to be sent to the provider. - -The `StreamBatchEventHook` will be called for each event received from the provider and each event that is going to be sent to the provider, making it possible to rewrite the event data to a format usable within Cosmo streams or by external systems. +Add two new hooks to the stream lifecycle: `StreamBatchEventHook` and `StreamPublishEventHook`. +The `StreamBatchEventHook` will be called each time a batch of events is received from the provider, making it possible to rewrite, filter or split the event data to a format usable within Cosmo streams. +The `StreamPublishEventHook` will be called each time a batch of events is going to be sent to the provider, making it possible to rewrite, filter or split the event data to a format usable by external systems. The hook arguments are: * `ctx StreamBatchEventHookContext`: The stream context, which contains the ID and type of the stream (inbound or outbound) @@ -430,7 +408,7 @@ The hook will also return an error if one of the events cannot be processed, pre I also considered exposing the subscription context to the hook, but this would be too easy to misuse. Users might add subscription-specific data to an event that will be sent to multiple providers. To make it safe, I would need to copy the entire event data for each pair of event and subscription that needs to receive it. -#### Do we need a new hook? +#### Do we need two new hooks? Another possible solution for mapping outward data would be to use the existing middleware hooks `RouterOnRequestHandler` or `RouterMiddlewareHandler` to intercept the mutation, access the stream context, and emit the event to the stream. However, this would require exposing a stream context in the request lifecycle, which is difficult. It would also require coordination to ensure that an event emitted on the stream is sent only after the subscription starts. @@ -457,26 +435,11 @@ type SubscriptionEventConfiguration interface { ProviderID() string } -type StreamBatchDirection string - -const ( - StreamBatchDirectionInbound StreamBatchDirection = "inbound" - StreamBatchDirectionOutbound StreamBatchDirection = "outbound" -) - type StreamBatchEventHookContext interface { - Direction() StreamBatchDirection RequestContext() RequestContext } -// each provider will have its own event type that implements the StreamEvent interface -type NatsEvent struct { - ProviderID string - Subject string - Data json.RawMessage - Metadata map[string]string -} -// StreamBatchEventHook processes a batch of stream events (inbound or outbound). +// StreamBatchEventHook processes a batch of inbound stream events. // // Return: // - empty slice: drop all events. @@ -486,14 +449,17 @@ type StreamBatchEventHook interface { OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) } +// each provider will have its own event type that implements the StreamEvent interface +type NatsEvent struct { + ProviderID string + Subject string + Data json.RawMessage + Metadata map[string]string +} + type MyModule struct {} func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) { - // we only filter the events for inbound events - if ctx.Direction() == StreamBatchDirectionOutbound { - return events, nil - } - newEvents := make([]StreamEvent, 0, len(events)) clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { @@ -532,7 +498,7 @@ func (m *MyModule) Module() core.ModuleInfo { We can use the new `StreamBatchEventHook` to filter events based on the stream configuration and the client's scopes. The hook arguments are: -* `ctx StreamBatchEventHookContext`: The stream context, which contains the ID and type of the stream (inbound or outbound) and the request context +* `ctx StreamBatchEventHookContext`: The stream context, which contains the ID of the stream and the request context * `events []StreamEvent`: The events received from the provider or the events that are going to be sent to the provider The hook will return a new slice of events that will be used to emit the events to the client or to the provider. @@ -562,7 +528,7 @@ One or more batched events are received from the provider One or more batched events are sent to the provider │ - └─▶ core.StreamBatchEventHook (Data mapping, Filtering) + └─▶ core.StreamPublishEventHook (Data mapping, Filtering) │ └─▶ "Send event to provider" ``` @@ -581,3 +547,289 @@ The implementation of this solution will only require changes in the Cosmo repos - All hook implementations could raise a panic, so we need to implement proper error handling - Especially the casting of the event to the specific type for the provider could raise a panic if the event is not of the expected type and the developer is not using the type check - We should add metrics to track how much time is spent in each hook, to help customers identify slow hooks + +## Development workflow of subscription with custom modules + +Lets build an example of how the development workflow would look like for a developer that want to add a custom module to the cosmo streams engine. The idea is to build a module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. + +### 1. Add a subscription to the cosmo streams graphql schema + +The developer will start by adding a subscription to the cosmo streams graphql schema. +```graphql +type Subscription { + employeeUpdates(): Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") +} + +type Employee @key(fields: "id", resolvable: false) { + id: Int! @external +} +``` +After publishing the schema, the developer will need to add the module to the cosmo streams engine. + +### 2. Write the custom module + +The developer will need to write the custom module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. + +```go +package mymodule + +import ( + "encoding/json" + "slices" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" +) + +func init() { + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) +} + +type MyModule struct {} + +func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { + // check if the provider is nats + if ctx.StreamContext().ProviderType() != "nats" { + return events, nil + } + + // check if the client is allowed to subscribe to the stream + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + if !found { + return events, fmt.Errorf("client is not allowed to subscribe to the stream") + } + + newEvents := make([]core.StreamEvent, 0, len(events)) + + for _, evt := range events { + if natsEvent, ok := evt.(*nats.NatsEvent); ok { + // check if the subject is the one expected by the module + if natsEvent.Subject != "employeeUpdates" { + newEvents = append(newEvents, evt) + continue + } + + // check if the provider id is the one expected by the module + if natsEvent.ProviderID != "my-nats" { + newEvents = append(newEvents, evt) + continue + } + + // decode the event data coming from the provider + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // filter the events based on the client's scopes + if !slices.Contains(clientAllowedEntitiesIds, dataReceived.EmployeeId) { + continue + } + + // prepare the data to send to the client + var dataToSend struct { + Id string `json:"id"` + TypeName string `json:"__typename"` + } + dataToSend.Id = dataReceived.EmployeeId + dataToSend.TypeName = "Employee" + + // marshal the data to send to the client + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + // create the new event + newEvent := &nats.NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataToSendMarshalled, + Metadata: natsEvent.Metadata, + } + newEvents = append(newEvents, newEvent) + } + } + return newEvents, nil +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} + +// Interface guards +var ( + _ core.StreamBatchEventHook = (*MyModule)(nil) +) +``` + +### 3. Add the provider configuration to the cosmo router +```yaml +version: "1" + +events: + providers: + nats: + - id: my-nats + url: "nats://localhost:4222" +``` + +### 4. Build the cosmo router with the custom module + +Build and run the router with the custom module added. + + + +## Development workflow of cosmo streams mutation with custom modules + +Lets build an example of how the development workflow would look like for a developer that want to add a custom module to the cosmo streams engine. The idea is to build a module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. + +### 1. Add a mutation to the cosmo streams graphql schema + +The developer will start by adding a mutation to the cosmo streams graphql schema. +```graphql +type Mutation { + updateEmployee(id: Int!, update: UpdateEmployeeInput!): edfs__PublishResult! @edfs__natsPublish(subject: "employeeUpdated", providerId: "my-nats") +} + +input UpdateEmployeeInput { + name: String + email: String +} +``` +After publishing the schema, the developer will need to add the module to the cosmo streams engine. + +### 2. Write the custom module + +The developer will need to write the custom module that will be used to publish the event to the `employeeUpdated` subject. It will also be used to validate if the client is allowed to publish the event and to remap the data to the expected format. + +```go +package mymodule + +import ( + "encoding/json" + "slices" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" +) + +func init() { + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) +} + +type MyModule struct {} + +func (m *MyModule) OnStreamPublish(ctx StreamPublishEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { + // check if the provider is nats + if ctx.StreamContext().ProviderType() != "nats" { + return events, nil + } + + // check if the client is allowed to publish the event + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + if !found { + return events, fmt.Errorf("client is not allowed to publish the event") + } + + newEvents := make([]core.StreamEvent, 0, len(events)) + + for _, evt := range events { + if natsEvent, ok := evt.(*nats.NatsEvent); ok { + // check if the subject is the one expected by the module + if natsEvent.Subject != "employeeUpdated" { + newEvents = append(newEvents, evt) + continue + } + + // check if the provider id is the one expected by the module + if natsEvent.ProviderID != "my-nats" { + newEvents = append(newEvents, evt) + continue + } + + // decode the event data coming from cosmo streams + var dataReceived struct { + Id string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // skip the event if the client is not allowed to publish the event + if !slices.Contains(clientAllowedEntitiesIds, dataReceived.Id) { + continue + } + + // prepare the data to send to the client + var dataToSend struct { + EmployeeId string `json:"employeeId"` + EmployeeName string `json:"employeeName"` + EmployeeEmail string `json:"employeeEmail"` + } + dataToSend.EmployeeId = dataReceived.Id + dataToSend.EmployeeName = dataReceived.Name + dataToSend.EmployeeEmail = dataReceived.Email + + // marshal the data to send to the client + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + // create the new event + newEvent := &nats.NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataToSendMarshalled, + Metadata: natsEvent.Metadata, + } + newEvents = append(newEvents, newEvent) + } + } + return newEvents, nil +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} + +// Interface guards +var ( + _ core.StreamPublishEventHook = (*MyModule)(nil) +) +``` + +### 3. Add the provider configuration to the cosmo router +```yaml +version: "1" + +events: + providers: + nats: + - id: my-nats + url: "nats://localhost:4222" +``` + +### 4. Build the cosmo router with the custom module + +Build and run the router with the custom module added. From 23f3e19de80fdb95ef674afe9293796245126a4d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 12:11:01 +0200 Subject: [PATCH 015/173] chore: add some comments --- rfc/cosmo-streams-v1.md | 46 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 84536e663e..c8be82cefa 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -56,6 +56,7 @@ type NatsSubscriptionEventConfiguration struct { type MyModule struct {} +// This is a custom function that will be used to check if the client is allowed to subscribe to the stream func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) bool { cfg, ok := ctx.StreamContext().SubscriptionConfiguration().(*NatsSubscriptionEventConfiguration) if !ok { @@ -81,8 +82,11 @@ func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) return false } +// This is the new hook that will be called once at subscription start func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { + // check if the client is allowed to subscribe to the stream if !customCheckIfClientIsAllowedToSubscribe(ctx) { + // if not, return an error to prevent the subscription from starting return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") } return nil @@ -165,10 +169,15 @@ type NatsEvent struct { type MyModule struct {} +// This is the new hook that will be called once at subscription start func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { + // get the operation name and variables that we need opName := ctx.RequestContext().Operation().Name() opVarId := ctx.RequestContext().Operation().Variables().GetInt("id") + + // check if the operation name is the one expected by the module if opName == "employeeSub" { + // create the event to emit using the operation variables evt := &NatsEvent{ ProviderID: "employee-stream", Subject: "employee-stream", @@ -177,6 +186,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error "entity-id": fmt.Sprintf("%d", opVarId), }, } + // emit the event to the stream, that will be received by the client ctx.StreamContext().WriteEvent(evt) } return nil @@ -245,15 +255,19 @@ type StreamBatchEventHook interface { type MyModule struct {} +// This is the new hook that will be called each time a batch of events is received from the provider func (m *MyModule) OnStreamEvents( ctx StreamBatchEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { + // create a new slice of events that we will return with the events with the new format newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { + // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { + // check if the subject is the one expected by the module if natsEvent.Subject == "topic-with-internal-data-format" { - // rewrite the event data to a format that is usable by Cosmo streams + // unmarshal the event data that we received from the provider var dataReceived struct { EmployeeId string `json:"EmployeeId"` } @@ -261,6 +275,8 @@ func (m *MyModule) OnStreamEvents( if err != nil { return events, fmt.Errorf("error unmarshalling data: %w", err) } + + // prepare the data to send to the client var dataForStream struct { Id string `json:"id"` Name string `json:"__typename"` @@ -268,21 +284,25 @@ func (m *MyModule) OnStreamEvents( dataForStream.Id = dataReceived.EmployeeId dataForStream.Name = "Employee" + // marshal the data to send to the client dataForStreamMarshalled, err := json.Marshal(dataForStream) if err != nil { return events, fmt.Errorf("error marshalling data: %w", err) } + // create the new event newEvent := &NatsEvent{ ProviderID: natsEvent.ProviderID, Subject: natsEvent.Subject, Data: dataForStreamMarshalled, Metadata: natsEvent.Metadata, } + // add the new event to the slice of events to return newEvents = append(newEvents, newEvent) continue } } + // add the original event to the slice of events to return newEvents = append(newEvents, evt) } @@ -335,15 +355,19 @@ type NatsEvent struct { type MyModule struct {} +// This is the new hook that will be called each time a batch of events is going to be sent to the provider func (m *MyModule) OnPublishEvents( ctx StreamPublishEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { + // create a new slice of events that we will return with the events with the new format newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { + // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { + // check if the subject is the one expected by the module if natsEvent.Subject == "topic-with-internal-data-format" { - // unmarshal the event data that we received from the provider + // unmarshal the event data that we received from cosmo streams var dataReceived struct { Id string `json:"id"` TypeName string `json:"__typename"` @@ -364,6 +388,8 @@ func (m *MyModule) OnPublishEvents( if err != nil { return events, fmt.Errorf("error marshalling data: %w", err) } + + // create the new event newEvent := &NatsEvent{ ProviderID: natsEvent.ProviderID, Subject: natsEvent.Subject, @@ -373,6 +399,8 @@ func (m *MyModule) OnPublishEvents( "entity-domain": "employee", }, } + + // add the new event to the slice of events to return newEvents = append(newEvents, newEvent) continue } @@ -400,7 +428,7 @@ The `StreamBatchEventHook` will be called each time a batch of events is receive The `StreamPublishEventHook` will be called each time a batch of events is going to be sent to the provider, making it possible to rewrite, filter or split the event data to a format usable by external systems. The hook arguments are: -* `ctx StreamBatchEventHookContext`: The stream context, which contains the ID and type of the stream (inbound or outbound) +* `ctx StreamBatchEventHookContext`: The stream context, which contains the provider ID * `events []StreamEvent`: The events received from the provider or the events that are going to be sent to the provider The hook will return a new slice of events that will be used to emit the events to the client or to the provider. @@ -459,21 +487,31 @@ type NatsEvent struct { type MyModule struct {} +// This is the new hook that will be called each time a batch of events is received from the provider func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) { + // create a new slice of events that we will return with the events that are allowed to be received by the client newEvents := make([]StreamEvent, 0, len(events)) + + // get the client's allowed entities IDs clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { + // if the client doesn't have allowed entities IDs, return the original events return newEvents, nil } for _, evt := range events { + // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { + // check if the subject is the one expected by the module if natsEvent.Subject == "topic-with-internal-data-format" { + // check the entity ID in the metadata idHeader, ok := natsEvent.Metadata["entity-id"] if !ok { continue } + // check if the entity ID is in the client's allowed entities IDs if slices.Contains(clientAllowedEntitiesIds, idHeader) { + // add the event to the slice of events to return because the client is allowed to receive it newEvents = append(newEvents, evt) } } @@ -506,7 +544,7 @@ The hook will also return an error if one of the events cannot be processed, pre ## Architecture -With this proposal, we will add a new hook to the subscription and stream lifecycles. +With this proposal, we will add two new hooks to stream lifecycles and other hooks to the subscription lifecycle. ### Subscription Lifecycle ``` From 4d76c66afe0e713e5436310f7ae53b037e84fe85 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 12:26:32 +0200 Subject: [PATCH 016/173] chore: moved interfaces and types to an appendix --- rfc/cosmo-streams-v1.md | 169 +++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 80 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index c8be82cefa..12acfb6dc3 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -23,24 +23,8 @@ We already allow some customization using `RouterOnRequestHandler`, but it has n ### Example: Check if the client is allowed to subscribe to the stream ```go -// the structs are reported only with the fields that are used in the example -type SubscriptionEventConfiguration interface { - ProviderID() string -} - -type StreamContext interface { - ProviderType() string - SubscriptionConfiguration() SubscriptionEventConfiguration -} - -type RequestContext interface { - Authentication() *core.Authentication -} - -type SubscriptionOnStartHookContext interface { - RequestContext() RequestContext - StreamContext() StreamContext -} +// the interfaces/structs are reported partially to make the example more readable +// the full new interfaces/structs are available in the appendix 1 // This is the new hook that will be called once at subscription start type SubscriptionOnStartHandler interface { @@ -128,31 +112,8 @@ To emit an initial message on subscription start, we need access to the stream c ### Example ```go -// the structs are reported only with the fields that are used in the example -type StreamEvent interface { - Data() []byte - SetData(data []byte) -} - -type StreamContext interface { - ProviderType() string - WriteEvent(event core.StreamEvent) -} - -type OperationContext interface { - Name() string - // the variables are currently not available, so we need to add them here - Variables() *astjson.Value -} - -type RequestContext interface { - Operation() core.OperationContext -} - -type SubscriptionOnStartHookContext struct { - RequestContext() RequestContext - StreamContext() StreamContext -} +// the interfaces/structs are reported partially to make the example more readable +// the full new interfaces/structs are available in the appendix 1 // This is the new hook that will be called once at stream start type SubscriptionOnStartHandler interface { @@ -222,18 +183,8 @@ The current approach for emitting and reading data from the stream is not flexib ### Example 1: Rewrite the event received from the provider to a format that is usable by Cosmo streams ```go -// the structs are reported only with the fields that are used in the example -type StreamEvent interface { - Data() []byte - SetData(data []byte) -} - -type SubscriptionEventConfiguration interface { - ProviderID() string -} - -type StreamBatchEventHookContext interface { -} +// the interfaces/structs are reported partially to make the example more readable +// the full new interfaces/structs are available in the appendix 1 // each provider will have its own event type that implements the StreamEvent interface type NatsEvent struct { @@ -323,17 +274,8 @@ func (m *MyModule) Module() core.ModuleInfo { ### Example 2: Rewrite the event before emitting it to the provider to a format that is usable by external systems ```go -// the structs are reported only with the fields that are used in the example -type StreamEvent interface { - Data() []byte - SetData(data []byte) -} - -type SubscriptionEventConfiguration interface { - ProviderID() string -} - -type StreamPublishEventHookContext interface {} +// the interfaces/structs are reported partially to make the example more readable +// the full new interfaces/structs are available in the appendix 1 // StreamPublishEventHook processes a batch of outbound stream events // @@ -342,7 +284,7 @@ type StreamPublishEventHookContext interface {} // - non-empty slice: emit those events (can grow, shrink, or reorder the batch). // err != nil: abort the subscription with an error. type StreamPublishEventHook interface { - OnPublishEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) + OnPublishEvents(ctx StreamPublishEventHookContext, events []StreamEvent) ([]StreamEvent, error) } // each provider will have its own event type that implements the StreamEvent interface @@ -453,19 +395,8 @@ We need to allow customers to filter events based on custom logic. We currently ### Example: Filter events based on stream configuration and client's scopes ```go -// the structs are reported only with the fields that are used in the example -type StreamEvent interface { - Data() []byte - SetData(data []byte) -} - -type SubscriptionEventConfiguration interface { - ProviderID() string -} - -type StreamBatchEventHookContext interface { - RequestContext() RequestContext -} +// the interfaces/structs are reported partially to make the example more readable +// the full new interfaces/structs are available in the appendix 1 // StreamBatchEventHook processes a batch of inbound stream events. // @@ -871,3 +802,81 @@ events: ### 4. Build the cosmo router with the custom module Build and run the router with the custom module added. + +## Appendix 1, new data structures + +```go +// NEW HOOKS + +// SubscriptionOnStartHandler is a hook that is called once at subscription start +// it is used to validate if the client is allowed to subscribe to the stream +// if returns an error, the subscription will not start +type SubscriptionOnStartHandler interface { + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error +} + +// StreamBatchEventHook processes a batch of inbound stream events +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamBatchEventHook interface { + OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} + +// StreamPublishEventHook processes a batch of outbound stream events +// +// Return: +// - empty slice: drop all events. +// - non-empty slice: emit those events (can grow, shrink, or reorder the batch). +// err != nil: abort the subscription with an error. +type StreamPublishEventHook interface { + OnPublishEvents(ctx StreamPublishEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} + +// NEW INTERFACES +type SubscriptionEventConfiguration interface { + ProviderID() string +} + +type StreamEvent interface { + Data() []byte + SetData(data []byte) +} + +type StreamBatchEventHookContext interface { + ProviderType() string + ProviderID() string + RequestContext() RequestContext + SubscriptionConfiguration() SubscriptionEventConfiguration +} + +type StreamPublishEventHookContext interface { + ProviderType() string + ProviderID() string + RequestContext() RequestContext +} + +type SubscriptionOnStartHookContext interface { + ProviderType() string + SubscriptionConfiguration() SubscriptionEventConfiguration + WriteEvent(event core.StreamEvent) +} + +type RequestContext interface { + Authentication() *core.Authentication +} + +type SubscriptionOnStartHookContext interface { + RequestContext() RequestContext + StreamContext() StreamContext +} + +// ALREADY EXISTING INTERFACES THAT WILL BE UPDATED +type OperationContext interface { + Name() string + // the variables are currently not available, so we need to add them here + Variables() *astjson.Value +} +``` \ No newline at end of file From e35ac472ab97f3b9e02b020250009012e062a032 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 17:00:36 +0200 Subject: [PATCH 017/173] chore: improve example checks --- rfc/cosmo-streams-v1.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 12acfb6dc3..faafdcad97 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -135,12 +135,22 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error // get the operation name and variables that we need opName := ctx.RequestContext().Operation().Name() opVarId := ctx.RequestContext().Operation().Variables().GetInt("id") + + // check if the provider ID is the one expected by the module + if ctx.StreamContext().ProviderID() != "my-provider-id" { + return fmt.Errorf("provider ID is not the one expected by the module") + } + + //check if the provider type is the one expected by the module + if ctx.StreamContext().ProviderType() != "nats" { + return fmt.Errorf("provider type is not the one expected by the module") + } // check if the operation name is the one expected by the module if opName == "employeeSub" { // create the event to emit using the operation variables evt := &NatsEvent{ - ProviderID: "employee-stream", + ProviderID: ctx.StreamContext().ProviderID(), Subject: "employee-stream", Data: []byte(fmt.Sprintf("{\"id\": \"%d\", \"__typename\": \"Employee\"}", opVarId)), Metadata: map[string]string{ @@ -860,17 +870,11 @@ type StreamPublishEventHookContext interface { type SubscriptionOnStartHookContext interface { ProviderType() string - SubscriptionConfiguration() SubscriptionEventConfiguration - WriteEvent(event core.StreamEvent) -} - -type RequestContext interface { - Authentication() *core.Authentication -} - -type SubscriptionOnStartHookContext interface { + ProviderID() string RequestContext() RequestContext StreamContext() StreamContext + SubscriptionConfiguration() SubscriptionEventConfiguration + WriteEvent(event core.StreamEvent) } // ALREADY EXISTING INTERFACES THAT WILL BE UPDATED From df62784634756d5a611723125f9a80a3c7e53fad Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 17:38:07 +0200 Subject: [PATCH 018/173] chore: improve data structure to expose ProviderId/Type and Specific fields only where needed --- rfc/cosmo-streams-v1.md | 265 +++++++++++++++++++++++----------------- 1 file changed, 150 insertions(+), 115 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index faafdcad97..0b6d6a8fec 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -42,7 +42,7 @@ type MyModule struct {} // This is a custom function that will be used to check if the client is allowed to subscribe to the stream func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) bool { - cfg, ok := ctx.StreamContext().SubscriptionConfiguration().(*NatsSubscriptionEventConfiguration) + cfg, ok := ctx.SubscriptionEventConfiguration().(*NatsSubscriptionEventConfiguration) if !ok { return true } @@ -101,7 +101,7 @@ The hook should return `nil` if the client is allowed to subscribe to the stream I evaluated the possibility of adding the `SubscriptionContext` to the request context and using it within one of the existing hooks, but it would be difficult to build the subscription context without executing the pubsub code. -The `StreamContext.SubscriptionConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. +The `StreamContext.SubscriptionEventConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. ## Initial Message @@ -122,8 +122,6 @@ type SubscriptionOnStartHandler interface { // each provider will have its own event type that implements the StreamEvent interface type NatsEvent struct { - ProviderID string - Subject string Data json.RawMessage Metadata map[string]string } @@ -137,21 +135,19 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error opVarId := ctx.RequestContext().Operation().Variables().GetInt("id") // check if the provider ID is the one expected by the module - if ctx.StreamContext().ProviderID() != "my-provider-id" { - return fmt.Errorf("provider ID is not the one expected by the module") + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-provider-id" { + return nil } //check if the provider type is the one expected by the module - if ctx.StreamContext().ProviderType() != "nats" { - return fmt.Errorf("provider type is not the one expected by the module") + if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + return nil } // check if the operation name is the one expected by the module if opName == "employeeSub" { // create the event to emit using the operation variables evt := &NatsEvent{ - ProviderID: ctx.StreamContext().ProviderID(), - Subject: "employee-stream", Data: []byte(fmt.Sprintf("{\"id\": \"%d\", \"__typename\": \"Employee\"}", opVarId)), Metadata: map[string]string{ "entity-id": fmt.Sprintf("%d", opVarId), @@ -198,8 +194,6 @@ The current approach for emitting and reading data from the stream is not flexib // each provider will have its own event type that implements the StreamEvent interface type NatsEvent struct { - ProviderID string - Subject string Data json.RawMessage Metadata map[string]string } @@ -221,47 +215,60 @@ func (m *MyModule) OnStreamEvents( ctx StreamBatchEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { + // check if the provider ID is the one expected by the module + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-provider-id" { + return events, nil + } + + // check if the provider type is the one expected by the module + if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "topic-with-internal-data-format" { + return events, nil + } + // create a new slice of events that we will return with the events with the new format newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { - // check if the subject is the one expected by the module - if natsEvent.Subject == "topic-with-internal-data-format" { - // unmarshal the event data that we received from the provider - var dataReceived struct { - EmployeeId string `json:"EmployeeId"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } - - // prepare the data to send to the client - var dataForStream struct { - Id string `json:"id"` - Name string `json:"__typename"` - } - dataForStream.Id = dataReceived.EmployeeId - dataForStream.Name = "Employee" - - // marshal the data to send to the client - dataForStreamMarshalled, err := json.Marshal(dataForStream) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } - - // create the new event - newEvent := &NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, - Data: dataForStreamMarshalled, - Metadata: natsEvent.Metadata, - } - // add the new event to the slice of events to return - newEvents = append(newEvents, newEvent) - continue + // unmarshal the event data that we received from the provider + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // prepare the data to send to the client + var dataForStream struct { + Id string `json:"id"` + Name string `json:"__typename"` + } + dataForStream.Id = dataReceived.EmployeeId + dataForStream.Name = "Employee" + + // marshal the data to send to the client + dataForStreamMarshalled, err := json.Marshal(dataForStream) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + // create the new event + newEvent := &NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataForStreamMarshalled, + Metadata: natsEvent.Metadata, + } + // add the new event to the slice of events to return + newEvents = append(newEvents, newEvent) + continue } // add the original event to the slice of events to return newEvents = append(newEvents, evt) @@ -299,8 +306,6 @@ type StreamPublishEventHook interface { // each provider will have its own event type that implements the StreamEvent interface type NatsEvent struct { - ProviderID string - Subject string Data json.RawMessage Metadata map[string]string } @@ -312,50 +317,63 @@ func (m *MyModule) OnPublishEvents( ctx StreamPublishEventHookContext, events []StreamEvent, ) ([]StreamEvent, error) { + // check if the provider ID is the one expected by the module + if ctx.PublishEventConfiguration().ProviderID() != "my-provider-id" { + return events, nil + } + + // check if the provider type is the one expected by the module + if ctx.PublishEventConfiguration().ProviderType() != "nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.PublishEventConfiguration().(*nats.PublishAndRequestEventConfiguration) + if natsConfig.Subject != "topic-with-internal-data-format" { + return events, nil + } + // create a new slice of events that we will return with the events with the new format newEvents := make([]StreamEvent, 0, len(events)) for _, evt := range events { // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { - // check if the subject is the one expected by the module - if natsEvent.Subject == "topic-with-internal-data-format" { - // unmarshal the event data that we received from cosmo streams - var dataReceived struct { - Id string `json:"id"` - TypeName string `json:"__typename"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } - - // prepare the data to send to the provider to be usable from external systems - var dataToSend struct { - EmployeeId string `json:"EmployeeId"` - OtherField string `json:"OtherField"` - } - dataToSend.EmployeeId = dataReceived.Id - dataToSend.OtherField = "Custom value" - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } - - // create the new event - newEvent := &NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, - Data: dataToSendMarshalled, - Metadata: map[string]string{ - "entity-id": dataReceived.Id, - "entity-domain": "employee", - }, - } - - // add the new event to the slice of events to return - newEvents = append(newEvents, newEvent) - continue + // unmarshal the event data that we received from cosmo streams + var dataReceived struct { + Id string `json:"id"` + TypeName string `json:"__typename"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // prepare the data to send to the provider to be usable from external systems + var dataToSend struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + dataToSend.EmployeeId = dataReceived.Id + dataToSend.OtherField = "Custom value" + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) } + + // create the new event + newEvent := &NatsEvent{ + ProviderID: natsEvent.ProviderID, + Subject: natsEvent.Subject, + Data: dataToSendMarshalled, + Metadata: map[string]string{ + "entity-id": dataReceived.Id, + "entity-domain": "employee", + }, + } + + // add the new event to the slice of events to return + newEvents = append(newEvents, newEvent) + continue } newEvents = append(newEvents, evt) } @@ -379,14 +397,19 @@ Add two new hooks to the stream lifecycle: `StreamBatchEventHook` and `StreamPub The `StreamBatchEventHook` will be called each time a batch of events is received from the provider, making it possible to rewrite, filter or split the event data to a format usable within Cosmo streams. The `StreamPublishEventHook` will be called each time a batch of events is going to be sent to the provider, making it possible to rewrite, filter or split the event data to a format usable by external systems. -The hook arguments are: -* `ctx StreamBatchEventHookContext`: The stream context, which contains the provider ID -* `events []StreamEvent`: The events received from the provider or the events that are going to be sent to the provider +The hook arguments of `StreamBatchEventHook` are: +* `ctx StreamBatchEventHookContext`: The stream context, which contains the provider ID and the subscription configuration +* `events []StreamEvent`: The events received from the provider -The hook will return a new slice of events that will be used to emit the events to the client or to the provider. -The hook will also return an error if one of the events cannot be processed, preventing the event from being processed. +The hook will return a new slice of events that will be used to emit the events to the client. +The hook will also return an error if one of the events cannot be processed, preventing the events from being processed. + +The hook arguments of `StreamPublishEventHook` are: +* `ctx StreamPublishEventHookContext`: The stream context, which contains the provider ID and the publish configuration +* `events []StreamEvent`: The events that are going to be sent to the provider -I also considered exposing the subscription context to the hook, but this would be too easy to misuse. Users might add subscription-specific data to an event that will be sent to multiple providers. To make it safe, I would need to copy the entire event data for each pair of event and subscription that needs to receive it. +The hook will return a new slice of events that will be used to emit the events to the provider. +The hook will also return an error if one of the events cannot be processed, preventing the events from being processed. #### Do we need two new hooks? @@ -420,8 +443,6 @@ type StreamBatchEventHook interface { // each provider will have its own event type that implements the StreamEvent interface type NatsEvent struct { - ProviderID string - Subject string Data json.RawMessage Metadata map[string]string } @@ -430,6 +451,22 @@ type MyModule struct {} // This is the new hook that will be called each time a batch of events is received from the provider func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) { + // check if the provider ID is the one expected by the module + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-provider-id" { + return events, nil + } + + // check if the provider type is the one expected by the module + if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "topic-with-internal-data-format" { + return events, nil + } + // create a new slice of events that we will return with the events that are allowed to be received by the client newEvents := make([]StreamEvent, 0, len(events)) @@ -443,18 +480,15 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []Stre for _, evt := range events { // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { - // check if the subject is the one expected by the module - if natsEvent.Subject == "topic-with-internal-data-format" { - // check the entity ID in the metadata - idHeader, ok := natsEvent.Metadata["entity-id"] - if !ok { - continue - } - // check if the entity ID is in the client's allowed entities IDs - if slices.Contains(clientAllowedEntitiesIds, idHeader) { - // add the event to the slice of events to return because the client is allowed to receive it - newEvents = append(newEvents, evt) - } + // check the entity ID in the metadata + idHeader, ok := natsEvent.Metadata["entity-id"] + if !ok { + continue + } + // check if the entity ID is in the client's allowed entities IDs + if slices.Contains(clientAllowedEntitiesIds, idHeader) { + // add the event to the slice of events to return because the client is allowed to receive it + newEvents = append(newEvents, evt) } } } @@ -848,6 +882,12 @@ type StreamPublishEventHook interface { // NEW INTERFACES type SubscriptionEventConfiguration interface { ProviderID() string + ProviderType() string +} + +type PublishEventConfiguration interface { + ProviderID() string + ProviderType() string } type StreamEvent interface { @@ -856,24 +896,19 @@ type StreamEvent interface { } type StreamBatchEventHookContext interface { - ProviderType() string - ProviderID() string RequestContext() RequestContext - SubscriptionConfiguration() SubscriptionEventConfiguration + SubscriptionEventConfiguration() SubscriptionEventConfiguration } type StreamPublishEventHookContext interface { - ProviderType() string - ProviderID() string RequestContext() RequestContext + PublishEventConfiguration() PublishEventConfiguration } type SubscriptionOnStartHookContext interface { - ProviderType() string - ProviderID() string RequestContext() RequestContext StreamContext() StreamContext - SubscriptionConfiguration() SubscriptionEventConfiguration + SubscriptionEventConfiguration() SubscriptionEventConfiguration WriteEvent(event core.StreamEvent) } From ace1c5c0e84e11fb724b9653d94454a07f4b11d0 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 18:04:58 +0200 Subject: [PATCH 019/173] chore: removed logic in the examples --- rfc/cosmo-streams-v1.md | 264 +++++++++++++++++----------------------- 1 file changed, 110 insertions(+), 154 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 0b6d6a8fec..ea6237e269 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -50,18 +50,7 @@ func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) providerId := cfg.ProviderID clientScopes := ctx.RequestContext().Authentication().Scopes() - if slices.Contains(clientScopes, "admin") { - return true - } - - if providerId == "sharable-data" { - return true - } - - if providerId == "almost-sharable-data" && - slices.Equal(cfg.Subjects, []string{"public"}) { - return true - } + // add checks here on client scopes, provider ID, etc. return false } @@ -236,34 +225,16 @@ func (m *MyModule) OnStreamEvents( for _, evt := range events { // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { - // unmarshal the event data that we received from the provider - var dataReceived struct { - EmployeeId string `json:"EmployeeId"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } - - // prepare the data to send to the client - var dataForStream struct { - Id string `json:"id"` - Name string `json:"__typename"` - } - dataForStream.Id = dataReceived.EmployeeId - dataForStream.Name = "Employee" - - // marshal the data to send to the client - dataForStreamMarshalled, err := json.Marshal(dataForStream) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } + // here you can umarshal the old data and map it to the new format + // for example: + // var dataReceived struct { + // EmployeeId string `json:"EmployeeId"` + // } + // err := json.Unmarshal(natsEvent.Data(), &dataReceived) // create the new event newEvent := &NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, - Data: dataForStreamMarshalled, + Data: newDataFormat, Metadata: natsEvent.Metadata, } // add the new event to the slice of events to return @@ -338,32 +309,15 @@ func (m *MyModule) OnPublishEvents( for _, evt := range events { // check if the event is the one expected by the module if natsEvent, ok := evt.(*NatsEvent); ok { - // unmarshal the event data that we received from cosmo streams - var dataReceived struct { - Id string `json:"id"` - TypeName string `json:"__typename"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } - - // prepare the data to send to the provider to be usable from external systems - var dataToSend struct { - EmployeeId string `json:"EmployeeId"` - OtherField string `json:"OtherField"` - } - dataToSend.EmployeeId = dataReceived.Id - dataToSend.OtherField = "Custom value" - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } + // here you can umarshal the old data and map it to the new format + // for example: + // var dataReceived struct { + // EmployeeId string `json:"EmployeeId"` + // } + // err := json.Unmarshal(natsEvent.Data(), &dataReceived) // create the new event newEvent := &NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, Data: dataToSendMarshalled, Metadata: map[string]string{ "entity-id": dataReceived.Id, @@ -606,6 +560,17 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core return events, nil } + // check if the provider id is the one expected by the module + if ctx.StreamContext().ProviderID() != "my-nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "employeeUpdates" { + return events, nil + } + // check if the client is allowed to subscribe to the stream clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { @@ -615,57 +580,47 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core newEvents := make([]core.StreamEvent, 0, len(events)) for _, evt := range events { - if natsEvent, ok := evt.(*nats.NatsEvent); ok { - // check if the subject is the one expected by the module - if natsEvent.Subject != "employeeUpdates" { - newEvents = append(newEvents, evt) - continue - } - - // check if the provider id is the one expected by the module - if natsEvent.ProviderID != "my-nats" { - newEvents = append(newEvents, evt) - continue - } + natsEvent, ok := evt.(*nats.NatsEvent); + if !ok { + newEvents = append(newEvents, evt) + continue + } - // decode the event data coming from the provider - var dataReceived struct { - EmployeeId string `json:"EmployeeId"` - OtherField string `json:"OtherField"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } + // decode the event data coming from the provider + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } - // filter the events based on the client's scopes - if !slices.Contains(clientAllowedEntitiesIds, dataReceived.EmployeeId) { - continue - } + // filter the events based on the client's scopes + if !slices.Contains(clientAllowedEntitiesIds, dataReceived.EmployeeId) { + continue + } - // prepare the data to send to the client - var dataToSend struct { - Id string `json:"id"` - TypeName string `json:"__typename"` - } - dataToSend.Id = dataReceived.EmployeeId - dataToSend.TypeName = "Employee" + // prepare the data to send to the client + var dataToSend struct { + Id string `json:"id"` + TypeName string `json:"__typename"` + } + dataToSend.Id = dataReceived.EmployeeId + dataToSend.TypeName = "Employee" - // marshal the data to send to the client - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } + // marshal the data to send to the client + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } - // create the new event - newEvent := &nats.NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, - Data: dataToSendMarshalled, - Metadata: natsEvent.Metadata, - } - newEvents = append(newEvents, newEvent) + // create the new event + newEvent := &nats.NatsEvent{ + Data: dataToSendMarshalled, + Metadata: natsEvent.Metadata, } + newEvents = append(newEvents, newEvent) } return newEvents, nil } @@ -749,6 +704,17 @@ func (m *MyModule) OnStreamPublish(ctx StreamPublishEventHookContext, events []c return events, nil } + // check if the provider id is the one expected by the module + if ctx.StreamContext().ProviderID() != "my-nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.PublishEventConfiguration().(*nats.PublishEventConfiguration) + if natsConfig.Subject != "employeeUpdated" { + return events, nil + } + // check if the client is allowed to publish the event clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { @@ -758,60 +724,50 @@ func (m *MyModule) OnStreamPublish(ctx StreamPublishEventHookContext, events []c newEvents := make([]core.StreamEvent, 0, len(events)) for _, evt := range events { - if natsEvent, ok := evt.(*nats.NatsEvent); ok { - // check if the subject is the one expected by the module - if natsEvent.Subject != "employeeUpdated" { - newEvents = append(newEvents, evt) - continue - } - - // check if the provider id is the one expected by the module - if natsEvent.ProviderID != "my-nats" { - newEvents = append(newEvents, evt) - continue - } + natsEvent, ok := evt.(*nats.NatsEvent); + if !ok { + newEvents = append(newEvents, evt) + continue + } - // decode the event data coming from cosmo streams - var dataReceived struct { - Id string `json:"id"` - Name string `json:"name"` - Email string `json:"email"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } + // decode the event data coming from cosmo streams + var dataReceived struct { + Id string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + } + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } - // skip the event if the client is not allowed to publish the event - if !slices.Contains(clientAllowedEntitiesIds, dataReceived.Id) { - continue - } + // skip the event if the client is not allowed to publish the event + if !slices.Contains(clientAllowedEntitiesIds, dataReceived.Id) { + continue + } - // prepare the data to send to the client - var dataToSend struct { - EmployeeId string `json:"employeeId"` - EmployeeName string `json:"employeeName"` - EmployeeEmail string `json:"employeeEmail"` - } - dataToSend.EmployeeId = dataReceived.Id - dataToSend.EmployeeName = dataReceived.Name - dataToSend.EmployeeEmail = dataReceived.Email - - // marshal the data to send to the client - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } + // prepare the data to send to the client + var dataToSend struct { + EmployeeId string `json:"employeeId"` + EmployeeName string `json:"employeeName"` + EmployeeEmail string `json:"employeeEmail"` + } + dataToSend.EmployeeId = dataReceived.Id + dataToSend.EmployeeName = dataReceived.Name + dataToSend.EmployeeEmail = dataReceived.Email + + // marshal the data to send to the client + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } - // create the new event - newEvent := &nats.NatsEvent{ - ProviderID: natsEvent.ProviderID, - Subject: natsEvent.Subject, - Data: dataToSendMarshalled, - Metadata: natsEvent.Metadata, - } - newEvents = append(newEvents, newEvent) + // create the new event + newEvent := &nats.NatsEvent{ + Data: dataToSendMarshalled, + Metadata: natsEvent.Metadata, } + newEvents = append(newEvents, newEvent) } return newEvents, nil } From e6fc78fdd0442df6eaa631211c83ad9da7df10f2 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 18:32:02 +0200 Subject: [PATCH 020/173] chore: add proposal about how we could integrate AsyncAPI --- rfc/cosmo-streams-v1.md | 260 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index ea6237e269..58cb5af930 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -874,4 +874,262 @@ type OperationContext interface { // the variables are currently not available, so we need to add them here Variables() *astjson.Value } -``` \ No newline at end of file +``` + +## Appendix 2, Using AsyncAPI for Event Data Structure + +As a side note, it is important to find ways to document the data that is arriving and going out of the cosmo streams engine. This could allow some automatic code generation starting from the schema and the events data. +As an example, we are going to explore how AsyncAPI could be used to generate the data structures for the custom modules and assure the messages format. + +### Example: AsyncAPI Integration for Custom Module Development + +We propose integrating AsyncAPI specifications with Cosmo streams to generate type-safe Go structs that can be used in custom modules. This would significantly improve the developer experience by providing: + +1. **Type Safety**: Generated structs prevent runtime errors from incorrect field access +2. **Documentation**: AsyncAPI specs serve as living documentation for event schemas +3. **Code Generation**: Automatic generation of Go structs from AsyncAPI specifications +4. **IDE Support**: Better autocomplete and error detection in development environments + +### AsyncAPI Specification Example + +So if we have as an example the following AsyncAPI specification: + +```yaml +# employee-events.asyncapi.yaml +asyncapi: 3.0.0 +info: + title: Employee Events API + version: 1.0.0 + description: Events related to employee updates in the system + +channels: + externalSystemEmployeeUpdates: + messages: + EmployeeUpdated: + $ref: '#/components/messages/EmployeeUpdated' + +components: + messages: + ExternalSystemEmployeeUpdated: + name: ExternalSystemEmployeeUpdated + title: External System Employee Updated Event + summary: Sent when an employee is updated in the external system + contentType: application/json + payload: + $ref: '#/components/schemas/ExternalSystemEmployeeFormat' + + schemas: + ExternalSystemEmployeeFormat: + type: object + description: Employee data as received from external systems + properties: + EmployeeId: + type: string + description: Unique identifier for the employee + EmployeeName: + type: string + description: Full name of the employee + EmployeeEmail: + type: string + format: email + description: Email address of the employee + OtherField: + type: string + description: Additional field from external system + required: + - EmployeeId + - EmployeeName + - EmployeeEmail +``` + +### Code Generation Workflow + +We could provide a CLI command to WGC to generate the Go structs from AsyncAPI specifications: + +```bash +# Generate Go structs from AsyncAPI spec +wgc streams generate -i employee-events.asyncapi.yaml -o ./generated/events.go -p events +``` + +Before generating the code, we could add to the data that cosmo streams is expecting to receive and send. +```yaml +# employee-events.asyncapi.yaml +asyncapi: 3.0.0 +info: + title: Cosmo Streams Employee Events API + version: 1.0.0 + +channels: + cosmoStreamsEmployeeUpdates: + messages: + CosmoStreamsEmployeeUpdated: + $ref: '#/components/messages/CosmoStreamsEmployeeUpdated' + +components: + messages: + CosmoStreamsEmployeeUpdated: + name: CosmoStreamsEmployeeUpdated + title: Cosmo Streams Employee Updated Event + summary: Event published when updating an employee in the cosmo streams + contentType: application/json + payload: + $ref: '#/components/schemas/EmployeeInternalFormat' + + schemas: + CosmoStreamsEmployeeUpdated: + type: object + description: Employee data as used internally by Cosmo streams + properties: + id: + type: string + description: Unique identifier for the employee + name: + type: string + description: Full name of the employee + email: + type: string + format: email + description: Email address of the employee + required: + - id + - __typename +``` + + +This command would be a wrapper around asyncapi modelina, and with some additional logic to extract the internal events format from the schema SDL. + +This would generate Go code like: + +```go +// generated/events.go +package events + +import ( + "encoding/json" + "time" +) + +// ExternalSystemEmployeeUpdated represents employee data as received from external systems +type ExternalSystemEmployeeUpdated struct { + EmployeeId string `json:"EmployeeId"` + EmployeeName string `json:"EmployeeName"` + EmployeeEmail string `json:"EmployeeEmail"` + OtherField string `json:"OtherField"` +} + +// EmployeeInternalFormat represents employee data as used internally by Cosmo streams +type CosmoStreamsEmployeeUpdated struct { + Id string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} +``` + +We could than encourage the developers to add conversions in a file in the same package of the generated file, like so: + +```go +// generated/events.go +package events + +import ( + "encoding/json" + "time" +) + +func ExternalSystemEmployeeUpdatedToCosmoStreamsEmployeeUpdated(e *ExternalSystemEmployeeUpdated) *CosmoStreamsEmployeeUpdated { + return &CosmoStreamsEmployeeUpdated{ + Id: e.EmployeeId, + Name: e.EmployeeName, + Email: e.EmployeeEmail, + } +} + +``` + + +### Enhanced Custom Module Development + +With generated structs, the custom module code becomes more maintainable and type-safe: + +```go +package mymodule + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" + "your-project/generated/genevents" +) + +type MyModule struct {} + +func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { + if ctx.StreamContext().ProviderType() != "nats" { + return events, nil + } + + if ctx.StreamContext().ProviderID() != "my-nats" { + return events, nil + } + + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "employeeUpdates" { + return events, nil + } + + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + if !found { + return events, fmt.Errorf("client is not allowed to subscribe to the stream") + } + + for _, evt := range events { + natsEvent, ok := evt.(*nats.NatsEvent); + if !ok { + newEvents = append(newEvents, evt) + continue + } + + // Use generated struct for type-safe deserialization + var dataReceived genevents.ExternalSystemEmployeeUpdated + err := json.Unmarshal(natsEvent.Data(), &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // Convert to internal format using generated method + dataToSend := genevents.ExternalSystemEmployeeUpdatedToCosmoStreamsEmployeeUpdated(&dataReceived) + + // Marshal using the generated struct + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + // Create new event + newEvent := &nats.NatsEvent{ + Data: dataToSendMarshalled, + } + newEvents = append(newEvents, newEvent) + } + return newEvents, nil +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} + +var _ core.StreamBatchEventHook = (*MyModule)(nil) +``` + +### Considerations + +The developers would need to regenerate the code each time the AsyncAPI specification changes or the schema SDL changes. \ No newline at end of file From 95a3941c7625fd86f247cab722d96ceaa50f68cf Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 18:41:37 +0200 Subject: [PATCH 021/173] core: add usage of FieldName() --- rfc/cosmo-streams-v1.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 58cb5af930..58c106734f 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -42,6 +42,12 @@ type MyModule struct {} // This is a custom function that will be used to check if the client is allowed to subscribe to the stream func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) bool { + // check if the field name is the one expected by the module + if ctx.SubscriptionEventConfiguration().FieldName() != "employeeUpdates" { + return true + } + + // get the specific configuration for the provider to make more advanced checks cfg, ok := ctx.SubscriptionEventConfiguration().(*NatsSubscriptionEventConfiguration) if !ok { return true @@ -839,11 +845,13 @@ type StreamPublishEventHook interface { type SubscriptionEventConfiguration interface { ProviderID() string ProviderType() string + FieldName() string // the field name of the subscription in the schema } type PublishEventConfiguration interface { ProviderID() string ProviderType() string + FieldName() string // the field name of the mutation in the schema } type StreamEvent interface { From 5adb360df121059c2ff22e553999fcb227c5849f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 19:28:31 +0200 Subject: [PATCH 022/173] chore: remove second example --- rfc/cosmo-streams-v1.md | 151 +--------------------------------------- 1 file changed, 3 insertions(+), 148 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 58c106734f..df376945e4 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -525,7 +525,9 @@ The implementation of this solution will only require changes in the Cosmo repos Lets build an example of how the development workflow would look like for a developer that want to add a custom module to the cosmo streams engine. The idea is to build a module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. -### 1. Add a subscription to the cosmo streams graphql schema +I'll show the workflow for a developer that wants to customize the subscription, but the same workflow can be applied to the mutation. + +### Add a subscription to the cosmo streams graphql schema The developer will start by adding a subscription to the cosmo streams graphql schema. ```graphql @@ -662,153 +664,6 @@ events: Build and run the router with the custom module added. - - -## Development workflow of cosmo streams mutation with custom modules - -Lets build an example of how the development workflow would look like for a developer that want to add a custom module to the cosmo streams engine. The idea is to build a module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. - -### 1. Add a mutation to the cosmo streams graphql schema - -The developer will start by adding a mutation to the cosmo streams graphql schema. -```graphql -type Mutation { - updateEmployee(id: Int!, update: UpdateEmployeeInput!): edfs__PublishResult! @edfs__natsPublish(subject: "employeeUpdated", providerId: "my-nats") -} - -input UpdateEmployeeInput { - name: String - email: String -} -``` -After publishing the schema, the developer will need to add the module to the cosmo streams engine. - -### 2. Write the custom module - -The developer will need to write the custom module that will be used to publish the event to the `employeeUpdated` subject. It will also be used to validate if the client is allowed to publish the event and to remap the data to the expected format. - -```go -package mymodule - -import ( - "encoding/json" - "slices" - "github.com/wundergraph/cosmo/router/core" - "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" -) - -func init() { - // Register your module here and it will be loaded at router start - core.RegisterModule(&MyModule{}) -} - -type MyModule struct {} - -func (m *MyModule) OnStreamPublish(ctx StreamPublishEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { - // check if the provider is nats - if ctx.StreamContext().ProviderType() != "nats" { - return events, nil - } - - // check if the provider id is the one expected by the module - if ctx.StreamContext().ProviderID() != "my-nats" { - return events, nil - } - - // check if the subject is the one expected by the module - natsConfig := ctx.PublishEventConfiguration().(*nats.PublishEventConfiguration) - if natsConfig.Subject != "employeeUpdated" { - return events, nil - } - - // check if the client is allowed to publish the event - clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] - if !found { - return events, fmt.Errorf("client is not allowed to publish the event") - } - - newEvents := make([]core.StreamEvent, 0, len(events)) - - for _, evt := range events { - natsEvent, ok := evt.(*nats.NatsEvent); - if !ok { - newEvents = append(newEvents, evt) - continue - } - - // decode the event data coming from cosmo streams - var dataReceived struct { - Id string `json:"id"` - Name string `json:"name"` - Email string `json:"email"` - } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) - if err != nil { - return events, fmt.Errorf("error unmarshalling data: %w", err) - } - - // skip the event if the client is not allowed to publish the event - if !slices.Contains(clientAllowedEntitiesIds, dataReceived.Id) { - continue - } - - // prepare the data to send to the client - var dataToSend struct { - EmployeeId string `json:"employeeId"` - EmployeeName string `json:"employeeName"` - EmployeeEmail string `json:"employeeEmail"` - } - dataToSend.EmployeeId = dataReceived.Id - dataToSend.EmployeeName = dataReceived.Name - dataToSend.EmployeeEmail = dataReceived.Email - - // marshal the data to send to the client - dataToSendMarshalled, err := json.Marshal(dataToSend) - if err != nil { - return events, fmt.Errorf("error marshalling data: %w", err) - } - - // create the new event - newEvent := &nats.NatsEvent{ - Data: dataToSendMarshalled, - Metadata: natsEvent.Metadata, - } - newEvents = append(newEvents, newEvent) - } - return newEvents, nil -} - -func (m *MyModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - ID: myModuleID, - Priority: 1, - New: func() core.Module { - return &MyModule{} - }, - } -} - -// Interface guards -var ( - _ core.StreamPublishEventHook = (*MyModule)(nil) -) -``` - -### 3. Add the provider configuration to the cosmo router -```yaml -version: "1" - -events: - providers: - nats: - - id: my-nats - url: "nats://localhost:4222" -``` - -### 4. Build the cosmo router with the custom module - -Build and run the router with the custom module added. - ## Appendix 1, new data structures ```go From bfe26512dd06100da12d2bd1e9b35c7a6520c2b6 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 19:36:21 +0200 Subject: [PATCH 023/173] chore: explicit that the second async schema can be used by outside systems. --- rfc/cosmo-streams-v1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index df376945e4..bf342fa628 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -816,7 +816,7 @@ wgc streams generate -i employee-events.asyncapi.yaml -o ./generated/events.go - Before generating the code, we could add to the data that cosmo streams is expecting to receive and send. ```yaml -# employee-events.asyncapi.yaml +# cosmo-streams-events.asyncapi.yaml asyncapi: 3.0.0 info: title: Cosmo Streams Employee Events API @@ -858,10 +858,9 @@ components: - __typename ``` - This command would be a wrapper around asyncapi modelina, and with some additional logic to extract the internal events format from the schema SDL. -This would generate Go code like: +This would generate a second async api specification and Go code like: ```go // generated/events.go @@ -909,6 +908,7 @@ func ExternalSystemEmployeeUpdatedToCosmoStreamsEmployeeUpdated(e *ExternalSyste ``` +Also, external systems could use the generated async api specification to generate the code for the events that they are sending/receiving to/from cosmo streams. ### Enhanced Custom Module Development From 368413eb7c4defb70621039efff2dab45563fbbe Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 19:38:26 +0200 Subject: [PATCH 024/173] chore: change sent to published to be more consistent --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index bf342fa628..6b81d8b633 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -499,7 +499,7 @@ One or more batched events are received from the provider │ └─▶ "Deliver events to client" -One or more batched events are sent to the provider +One or more batched events are published to the provider │ └─▶ core.StreamPublishEventHook (Data mapping, Filtering) │ From 9015adf3797cd1d4e95e428da3de460a301270dc Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 19:47:46 +0200 Subject: [PATCH 025/173] chore: add enums for provider types --- rfc/cosmo-streams-v1.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 6b81d8b633..a34783429e 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -135,7 +135,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } //check if the provider type is the one expected by the module - if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return nil } @@ -216,7 +216,7 @@ func (m *MyModule) OnStreamEvents( } // check if the provider type is the one expected by the module - if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } @@ -300,7 +300,7 @@ func (m *MyModule) OnPublishEvents( } // check if the provider type is the one expected by the module - if ctx.PublishEventConfiguration().ProviderType() != "nats" { + if ctx.PublishEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } @@ -417,7 +417,7 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []Stre } // check if the provider type is the one expected by the module - if ctx.SubscriptionEventConfiguration().ProviderType() != "nats" { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } @@ -564,7 +564,7 @@ type MyModule struct {} func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { // check if the provider is nats - if ctx.StreamContext().ProviderType() != "nats" { + if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { return events, nil } @@ -737,6 +737,15 @@ type OperationContext interface { // the variables are currently not available, so we need to add them here Variables() *astjson.Value } + +// STRUCTURES TO BE ADDED TO PUBSUB PACKAGE +type ProviderType string +const ( + ProviderTypeNats ProviderType = "nats" + ProviderTypeKafka ProviderType = "kafka" + ProviderTypeRedis ProviderType = "redis" +} + ``` ## Appendix 2, Using AsyncAPI for Event Data Structure From cc83382dbe1c15d1d0915d82d51a3b64429a99dd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sun, 13 Jul 2025 20:01:22 +0200 Subject: [PATCH 026/173] chore: remove Data and SetData, they break immutability! --- rfc/cosmo-streams-v1.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index a34783429e..33a87649c3 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -171,7 +171,7 @@ Using the new `SubscriptionOnStart` hook that we introduced for the previous req To emit the message, I propose adding a new method to the stream context, `WriteEvent`, which will emit the event to the stream at the lowest level. The message will pass through all hooks, making it behave like any other event received from the provider. -The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the event to the specific type for the provider. If the custom modules only need to read the data, they can use the `Data()`/`SetData()` methods without casting the event. +The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use events, the hook has to cast the event to the specific type for the provider. This change will require adding a new type in each provider package to represent the event with additional fields (metadata, etc.). This is a significant change, but it is necessary to support additional data in events, anyway, even if we don't expose them to the custom modules. @@ -236,7 +236,7 @@ func (m *MyModule) OnStreamEvents( // var dataReceived struct { // EmployeeId string `json:"EmployeeId"` // } - // err := json.Unmarshal(natsEvent.Data(), &dataReceived) + // err := json.Unmarshal(natsEvent.Data, &dataReceived) // create the new event newEvent := &NatsEvent{ @@ -320,7 +320,7 @@ func (m *MyModule) OnPublishEvents( // var dataReceived struct { // EmployeeId string `json:"EmployeeId"` // } - // err := json.Unmarshal(natsEvent.Data(), &dataReceived) + // err := json.Unmarshal(natsEvent.Data, &dataReceived) // create the new event newEvent := &NatsEvent{ @@ -599,7 +599,7 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core EmployeeId string `json:"EmployeeId"` OtherField string `json:"OtherField"` } - err := json.Unmarshal(natsEvent.Data(), &dataReceived) + err := json.Unmarshal(natsEvent.Data, &dataReceived) if err != nil { return events, fmt.Errorf("error unmarshalling data: %w", err) } @@ -709,10 +709,7 @@ type PublishEventConfiguration interface { FieldName() string // the field name of the mutation in the schema } -type StreamEvent interface { - Data() []byte - SetData(data []byte) -} +type StreamEvent interface {} type StreamBatchEventHookContext interface { RequestContext() RequestContext @@ -966,7 +963,7 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core // Use generated struct for type-safe deserialization var dataReceived genevents.ExternalSystemEmployeeUpdated - err := json.Unmarshal(natsEvent.Data(), &dataReceived) + err := json.Unmarshal(natsEvent.Data, &dataReceived) if err != nil { return events, fmt.Errorf("error unmarshalling data: %w", err) } From b5f188cd8634e13ba0f21d3940299edaf3891473 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:23:05 +0200 Subject: [PATCH 027/173] chore: FieldName -> RootFieldName --- rfc/cosmo-streams-v1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 33a87649c3..be789ad50b 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -43,7 +43,7 @@ type MyModule struct {} // This is a custom function that will be used to check if the client is allowed to subscribe to the stream func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) bool { // check if the field name is the one expected by the module - if ctx.SubscriptionEventConfiguration().FieldName() != "employeeUpdates" { + if ctx.SubscriptionEventConfiguration().RootFieldName() != "employeeUpdates" { return true } @@ -700,13 +700,13 @@ type StreamPublishEventHook interface { type SubscriptionEventConfiguration interface { ProviderID() string ProviderType() string - FieldName() string // the field name of the subscription in the schema + RootFieldName() string // the root field name of the subscription in the schema } type PublishEventConfiguration interface { ProviderID() string ProviderType() string - FieldName() string // the field name of the mutation in the schema + RootFieldName() string // the root field name of the mutation in the schema } type StreamEvent interface {} From 68e78749f988ce85e195f1bc6fd65271bb0580b5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:34:42 +0200 Subject: [PATCH 028/173] chore: improve data mapping description --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index be789ad50b..a7b3b02f65 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -3,7 +3,7 @@ Based on customer feedback, we've identified the need for more customizable stream behavior. The key areas for customization include: - **Authorization**: Implementing authorization checks at the start of subscriptions - **Initial message**: Sending an initial message to clients upon subscription start -- **Data mapping**: Transforming data to align with internal specifications +- **Data mapping**: Transforming events data from the format that could be used by the external system to/from Federation compatible Router events - **Event filtering**: Filtering events using custom logic Let's explore how we can address each of these requirements. From 127a08a50185212fb5a1935c6a5ae2c33dffdeee Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:37:41 +0200 Subject: [PATCH 029/173] chore: explicit that WriteEvent is writing event only to the current subscription and not going to other client that subscribed the same subscription --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index a7b3b02f65..c86e478f83 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -169,7 +169,7 @@ func (m *MyModule) Module() core.ModuleInfo { Using the new `SubscriptionOnStart` hook that we introduced for the previous requirement, we can emit the initial message on subscription start. We will also need access to operation variables, which are currently not available in the request context. -To emit the message, I propose adding a new method to the stream context, `WriteEvent`, which will emit the event to the stream at the lowest level. The message will pass through all hooks, making it behave like any other event received from the provider. +To emit the message, I propose adding a new method to the stream context, `WriteEvent`, which will emit the event to the stream at the lowest level. The message will pass through all hooks, making it behave like any other event received from the provider. The message will be received only by the client that subscribed to the stream, and not by the other clients that subscribed to the same stream. The `StreamEvent` contains the data as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use events, the hook has to cast the event to the specific type for the provider. From 083bf702f83f4165442cc8adb32a822ec3ecd9d5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:39:23 +0200 Subject: [PATCH 030/173] chore: explicit the WriteEvent behaviour also in the example --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index c86e478f83..9e06f2da0a 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -148,7 +148,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error "entity-id": fmt.Sprintf("%d", opVarId), }, } - // emit the event to the stream, that will be received by the client + // emit the event to the stream, that will be received only by the client that subscribed to the stream ctx.StreamContext().WriteEvent(evt) } return nil From ef36c76b1d848a69e74a827f90b792bdc8b5da23 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:51:56 +0200 Subject: [PATCH 031/173] chore: add an outlook on the Appendix 2 about a possible evolution --- rfc/cosmo-streams-v1.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 9e06f2da0a..08a9b00596 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -1001,4 +1001,10 @@ var _ core.StreamBatchEventHook = (*MyModule)(nil) ### Considerations -The developers would need to regenerate the code each time the AsyncAPI specification changes or the schema SDL changes. \ No newline at end of file +The developers would need to regenerate the code each time the AsyncAPI specification changes or the schema SDL changes. + +### Outlook + +In a second step, we could: +- allow the user to define their streams using AsyncAPI +- generate fully typesafe hooks with all events structures generated from the AsyncAPI specification \ No newline at end of file From 93e4b04c0da287a588baed64b3227b090604a6c5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 12:55:45 +0200 Subject: [PATCH 032/173] chore: fixes example code to avoid panic on request not authenticated --- rfc/cosmo-streams-v1.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 08a9b00596..6b63742962 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -430,6 +430,12 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []Stre // create a new slice of events that we will return with the events that are allowed to be received by the client newEvents := make([]StreamEvent, 0, len(events)) + + if ctx.RequestContext().Authentication() == nil { + // if the client is not authenticated, return no events + return newEvents, nil + } + // get the client's allowed entities IDs clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { @@ -579,6 +585,12 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core return events, nil } + // check if the client is authenticated + if ctx.RequestContext().Authentication() == nil { + // if the client is not authenticated, return no events + return events, nil + } + // check if the client is allowed to subscribe to the stream clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] if !found { From 1fdeec3687333f718c5ac069a79e4e74fd1a7f97 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 14:27:36 +0200 Subject: [PATCH 033/173] chore: add explicit reference to Filtering hook that could be used for authorization purposes --- rfc/cosmo-streams-v1.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 6b63742962..0f51cd210f 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -10,9 +10,10 @@ Let's explore how we can address each of these requirements. ## Authorization -To support authorization, we need a hook that enables two key decisions: +To support authorization, we need a hook that enables the following key decisions: - Whether the client or user is authorized to initiate the subscription - Which topics the client is permitted to subscribe to +- Whether the client is allowed to consume an event from the stream (covered by the Event Filtering hook) Additionally, a similar mechanism is required for non-stream subscriptions, allowing: - Custom JWT validation logic (e.g., expiration checks, signature verification, secret handling) From d38018e91b2f0486a1dce762630718ccd1079f31 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 14:32:05 +0200 Subject: [PATCH 034/173] chore: added description on the event filtering about authorization --- rfc/cosmo-streams-v1.md | 1 + 1 file changed, 1 insertion(+) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 0f51cd210f..5065dda8e0 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -385,6 +385,7 @@ Therefore, I believe the best solution is to add a new hooks to the stream lifec ## Event Filtering We need to allow customers to filter events based on custom logic. We currently only provide declarative filters, which are quite limited. +The event filtering hook will also be useful to implement the authorization logic at the events level. ### Example: Filter events based on stream configuration and client's scopes From f58970f153b597b9594c4546448eeac81ea5d840 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 14:51:23 +0200 Subject: [PATCH 035/173] chore: add StreamHookError to allow error response customization from the hook --- rfc/cosmo-streams-v1.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 5065dda8e0..30233ad8f3 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -39,6 +39,11 @@ type NatsSubscriptionEventConfiguration struct { StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` } +type StreamHookError struct { + HttpError core.HttpError + CloseSubscription bool +} + type MyModule struct {} // This is a custom function that will be used to check if the client is allowed to subscribe to the stream @@ -55,9 +60,9 @@ func customCheckIfClientIsAllowedToSubscribe(ctx SubscriptionOnStartHookContext) } providerId := cfg.ProviderID - clientScopes := ctx.RequestContext().Authentication().Scopes() + auth := ctx.RequestContext().Authentication() - // add checks here on client scopes, provider ID, etc. + // add checks here on client authentication scopes, provider ID, etc. return false } @@ -67,7 +72,13 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error // check if the client is allowed to subscribe to the stream if !customCheckIfClientIsAllowedToSubscribe(ctx) { // if not, return an error to prevent the subscription from starting - return fmt.Errorf("you should be an admin to subscribe to this or only subscribe to public subscriptions!") + return StreamHookError{ + HttpError: core.NewHttpGraphqlError( + "you should be an admin to subscribe to this or only subscribe to public subscriptions!", + "UNAUTHORIZED", + http.StatusUnauthorized, + ), CloseSubscription: true, + } } return nil } @@ -95,6 +106,8 @@ The hook arguments are: The hook should return an error if the client is not allowed to subscribe to the stream, preventing the subscription from starting. The hook should return `nil` if the client is allowed to subscribe to the stream, allowing the subscription to proceed. +The hook can return a `SubscriptionHookError` to customize the error messages and the behavior on the subscription. + I evaluated the possibility of adding the `SubscriptionContext` to the request context and using it within one of the existing hooks, but it would be difficult to build the subscription context without executing the pubsub code. The `StreamContext.SubscriptionEventConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. @@ -749,6 +762,13 @@ type OperationContext interface { Variables() *astjson.Value } +// NEW STRUCTURES +// StreamHookError is used to customize the error messages and the behavior +type StreamHookError struct { + HttpError core.HttpError + CloseSubscription bool +} + // STRUCTURES TO BE ADDED TO PUBSUB PACKAGE type ProviderType string const ( From 8ac629cd8138f38624e2aec4b143acc2cc610052 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 15:10:37 +0200 Subject: [PATCH 036/173] chore: add more details and an example with kafka --- rfc/cosmo-streams-v1.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 30233ad8f3..6041dcb893 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -195,6 +195,14 @@ Emitting the initial message with this hook ensures that the client will receive The current approach for emitting and reading data from the stream is not flexible enough. We need to be able to map data from an external format to the internal format, and vice versa. +Also, different providers can have different additional fields other than the message body. + +As an example: +- NATS provider can have a `Metadata` field +- Kafka provider can have a `Headers` and `Key` fields + +And this additional fields could be an important part of integrating with external systems. + ### Example 1: Rewrite the event received from the provider to a format that is usable by Cosmo streams ```go @@ -206,6 +214,11 @@ type NatsEvent struct { Data json.RawMessage Metadata map[string]string } +type KafkaEvent struct { + Key []byte + Data json.RawMessage + Headers map[[]byte]][]byte +} // StreamBatchEventHook processes a batch of inbound stream events // @@ -257,6 +270,14 @@ func (m *MyModule) OnStreamEvents( Data: newDataFormat, Metadata: natsEvent.Metadata, } + + // or for Kafka we would have something like: + // newEvent := &KafkaEvent{ + // Key: kafkaEvent.Key, + // Data: newDataFormat, + // Headers: kafkaEvent.Headers, + // } + // add the new event to the slice of events to return newEvents = append(newEvents, newEvent) continue From 434c61c9501d8609e3e5d8750706fee3ab9ce983 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 15:19:03 +0200 Subject: [PATCH 037/173] chore: add an example on how to use the metadata field --- rfc/cosmo-streams-v1.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 6041dcb893..9e0e604859 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -261,10 +261,20 @@ func (m *MyModule) OnStreamEvents( // here you can umarshal the old data and map it to the new format // for example: // var dataReceived struct { - // EmployeeId string `json:"EmployeeId"` + // EmployeeName string `json:"EmployeeName"` // } // err := json.Unmarshal(natsEvent.Data, &dataReceived) + // if we have to extract the data from the metadata fields, we can do it like this: + entityId := natsEvent.Metadata["entity-id"] + entityType := natsEvent.Metadata["entity-type"] + // and prepare the new event with the data inside + newDataFormat := json.Marshal(map[string]string{ + "id": entityId, + "name": dataReceived.EmployeeName, + "__typename": entityType, + }) + // create the new event newEvent := &NatsEvent{ Data: newDataFormat, @@ -277,7 +287,7 @@ func (m *MyModule) OnStreamEvents( // Data: newDataFormat, // Headers: kafkaEvent.Headers, // } - + // add the new event to the slice of events to return newEvents = append(newEvents, newEvent) continue From 35e80778b0d0f0c51ffcbb4676d17bf711a8aafa Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 14 Jul 2025 15:20:50 +0200 Subject: [PATCH 038/173] fix: ignore marshal error --- rfc/cosmo-streams-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 9e0e604859..9a32ad2e54 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -269,7 +269,7 @@ func (m *MyModule) OnStreamEvents( entityId := natsEvent.Metadata["entity-id"] entityType := natsEvent.Metadata["entity-type"] // and prepare the new event with the data inside - newDataFormat := json.Marshal(map[string]string{ + newDataFormat, _ := json.Marshal(map[string]string{ "id": entityId, "name": dataReceived.EmployeeName, "__typename": entityType, From fa562c964262a77a41ed993b5eb9ac2fb1aefb2b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 17:33:41 +0200 Subject: [PATCH 039/173] chore: add PublishEventConfiguration, ProviderType and specialized event types --- router-tests/.tool-versions | 1 + router/pkg/pubsub/datasource/provider.go | 27 +++++++++ router/pkg/pubsub/kafka/adapter.go | 8 +-- router/pkg/pubsub/kafka/engine_datasource.go | 57 +++++++++++++++++-- .../pubsub/kafka/engine_datasource_factory.go | 12 ++-- .../kafka/engine_datasource_factory_test.go | 2 +- .../pubsub/kafka/engine_datasource_test.go | 28 ++++----- router/pkg/pubsub/nats/adapter.go | 14 ++--- router/pkg/pubsub/nats/engine_datasource.go | 54 ++++++++++++++++-- .../pubsub/nats/engine_datasource_factory.go | 12 ++-- .../nats/engine_datasource_factory_test.go | 4 +- .../pkg/pubsub/nats/engine_datasource_test.go | 48 ++++++++-------- router/pkg/pubsub/redis/adapter.go | 8 +-- router/pkg/pubsub/redis/engine_datasource.go | 55 ++++++++++++++++-- .../pubsub/redis/engine_datasource_factory.go | 12 ++-- .../redis/engine_datasource_factory_test.go | 2 +- .../pubsub/redis/engine_datasource_test.go | 28 ++++----- 17 files changed, 269 insertions(+), 103 deletions(-) create mode 100644 router-tests/.tool-versions diff --git a/router-tests/.tool-versions b/router-tests/.tool-versions new file mode 100644 index 0000000000..99741b0b8c --- /dev/null +++ b/router-tests/.tool-versions @@ -0,0 +1 @@ +golang 1.23.9 diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index f90446a712..fb513a7fc7 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -31,3 +31,30 @@ type ProviderBuilder[P, E any] interface { // BuildEngineDataSourceFactory Build the data source for the given provider and event configuration BuildEngineDataSourceFactory(data E) (EngineDataSourceFactory, error) } + +// ProviderType represents the type of pubsub provider +type ProviderType string + +const ( + ProviderTypeNats ProviderType = "nats" + ProviderTypeKafka ProviderType = "kafka" + ProviderTypeRedis ProviderType = "redis" +) + +// StreamEvent is a generic interface for all stream events +// Each provider will have its own event type that implements this interface +type StreamEvent interface{} + +// SubscriptionEventConfiguration is the interface that all subscription event configurations must implement +type SubscriptionEventConfiguration interface { + ProviderID() string + ProviderType() ProviderType + RootFieldName() string // the root field name of the subscription in the schema +} + +// PublishEventConfiguration is the interface that all publish event configurations must implement +type PublishEventConfiguration interface { + ProviderID() string + ProviderType() ProviderType + RootFieldName() string // the root field name of the mutation in the schema +} diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 503b8f6f37..9ba0308582 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -99,7 +99,7 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "subscribe"), zap.Strings("topics", event.Topics), ) @@ -148,7 +148,7 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent // The event is written with a dedicated write client. func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "publish"), zap.String("topic", event.Topic), ) @@ -157,7 +157,7 @@ func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfigu return datasource.NewError("kafka write client not initialized", nil) } - log.Debug("publish", zap.ByteString("data", event.Data)) + log.Debug("publish", zap.ByteString("data", event.Event.Data)) var wg sync.WaitGroup wg.Add(1) @@ -166,7 +166,7 @@ func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfigu p.writeClient.Produce(ctx, &kgo.Record{ Topic: event.Topic, - Value: event.Data, + Value: event.Event.Data, }, func(record *kgo.Record, err error) { defer wg.Done() if err != nil { diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 7b82a766b0..62a83aadd9 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -9,25 +9,65 @@ import ( "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) +// Event represents an event from Kafka +type Event struct { + Key []byte `json:"key"` + Data json.RawMessage `json:"data"` + Headers map[string][]byte `json:"headers"` +} + type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Topics []string `json:"topics"` + ProviderID_ string `json:"providerId"` + Topics []string `json:"topics"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (s *SubscriptionEventConfiguration) ProviderID() string { + return s.ProviderID_ +} + +// ProviderType returns the provider type +func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeKafka +} + +// RootFieldName returns the root field name +func (s *SubscriptionEventConfiguration) RootFieldName() string { + return s.RootFieldName_ } type PublishEventConfiguration struct { - ProviderID string `json:"providerId"` - Topic string `json:"topic"` - Data json.RawMessage `json:"data"` + ProviderID_ string `json:"providerId"` + Topic string `json:"topic"` + Event Event `json:"event"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (p *PublishEventConfiguration) ProviderID() string { + return p.ProviderID_ +} + +// ProviderType returns the provider type +func (p *PublishEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeKafka +} + +// RootFieldName returns the root field name +func (p *PublishEventConfiguration) RootFieldName() string { + return p.RootFieldName_ } func (s *PublishEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"topic":"%s", "data": %s, "providerId":"%s"}`, s.Topic, s.Data, s.ProviderID) + return fmt.Sprintf(`{"topic":"%s", "data": %s, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID_) } type SubscriptionDataSource struct { @@ -86,3 +126,8 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B func (s *PublishDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) (err error) { panic("not implemented") } + +// Interface compliance checks +var _ datasource.SubscriptionEventConfiguration = (*SubscriptionEventConfiguration)(nil) +var _ datasource.PublishEventConfiguration = (*PublishEventConfiguration)(nil) +var _ datasource.StreamEvent = (*Event)(nil) diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index d360f02f26..e46250b4a8 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -49,9 +49,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri } evtCfg := PublishEventConfiguration{ - ProviderID: c.providerId, - Topic: c.topics[0], - Data: eventData, + ProviderID_: c.providerId, + Topic: c.topics[0], + Event: Event{Data: eventData}, + RootFieldName_: c.fieldName, } return evtCfg.MarshalJSONTemplate(), nil @@ -65,8 +66,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID: c.providerId, - Topics: c.topics, + ProviderID_: c.providerId, + Topics: c.topics, + RootFieldName_: c.fieldName, } object, err := json.Marshal(evtCfg) if err != nil { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go index 254359a4bc..c1bd6f0d56 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go @@ -33,7 +33,7 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // Configure mock expectations for Publish mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { - return event.ProviderID == "test-provider" && event.Topic == "test-topic" + return event.ProviderID() == "test-provider" && event.Topic == "test-topic" })).Return(nil) // Create the data source with mock adapter diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 0ad92aeb20..854db703f9 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -24,18 +24,18 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishEventConfiguration{ - ProviderID: "test-provider", - Topic: "test-topic", - Data: json.RawMessage(`{"message":"hello"}`), + ProviderID_: "test-provider", + Topic: "test-topic", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"topic":"test-topic", "data": {"message":"hello"}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishEventConfiguration{ - ProviderID: "test-provider-id", - Topic: "topic-with-hyphens", - Data: json.RawMessage(`{"message":"special \"quotes\" here"}`), + ProviderID_: "test-provider-id", + Topic: "topic-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"topic":"topic-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, }, @@ -113,8 +113,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Topics: []string{"topic1", "topic2"}, + ProviderID_: "test-provider", + Topics: []string{"topic1", "topic2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -124,8 +124,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"topics":["topic1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Topics: []string{"topic1"}, + ProviderID_: "test-provider", + Topics: []string{"topic1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, @@ -178,12 +178,12 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { }{ { name: "successful publish", - input: `{"topic":"test-topic", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { - return event.ProviderID == "test-provider" && + return event.ProviderID() == "test-provider" && event.Topic == "test-topic" && - string(event.Data) == `{"message":"hello"}` + string(event.Event.Data) == `{"message":"hello"}` })).Return(nil) }, expectError: false, @@ -192,7 +192,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { }, { name: "publish error", - input: `{"topic":"test-topic", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) }, diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index a0bef13f45..2b9a890498 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -74,7 +74,7 @@ func (p *ProviderAdapter) getDurableConsumerName(durableName string, subjects [] func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "subscribe"), zap.Strings("subjects", event.Subjects), ) @@ -199,7 +199,7 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent func (p *ProviderAdapter) Publish(_ context.Context, event PublishAndRequestEventConfiguration) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "publish"), zap.String("subject", event.Subject), ) @@ -208,9 +208,9 @@ func (p *ProviderAdapter) Publish(_ context.Context, event PublishAndRequestEven return datasource.NewError("nats client not initialized", nil) } - log.Debug("publish", zap.ByteString("data", event.Data)) + log.Debug("publish", zap.ByteString("data", event.Event.Data)) - err := p.client.Publish(event.Subject, event.Data) + err := p.client.Publish(event.Subject, event.Event.Data) if err != nil { log.Error("publish error", zap.Error(err)) return datasource.NewError(fmt.Sprintf("error publishing to NATS subject %s", event.Subject), err) @@ -221,7 +221,7 @@ func (p *ProviderAdapter) Publish(_ context.Context, event PublishAndRequestEven func (p *ProviderAdapter) Request(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "request"), zap.String("subject", event.Subject), ) @@ -230,9 +230,9 @@ func (p *ProviderAdapter) Request(ctx context.Context, event PublishAndRequestEv return datasource.NewError("nats client not initialized", nil) } - log.Debug("request", zap.ByteString("data", event.Data)) + log.Debug("request", zap.ByteString("data", event.Event.Data)) - msg, err := p.client.RequestWithContext(ctx, event.Subject, event.Data) + msg, err := p.client.RequestWithContext(ctx, event.Subject, event.Event.Data) if err != nil { log.Error("request error", zap.Error(err)) return datasource.NewError(fmt.Sprintf("error requesting from NATS subject %s", event.Subject), err) diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index ffc23ca838..a147c2f08f 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -9,10 +9,17 @@ import ( "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) +// NatsEvent represents an event from NATS +type Event struct { + Data json.RawMessage `json:"data"` + Metadata map[string]string `json:"metadata"` +} + type StreamConfiguration struct { Consumer string `json:"consumer"` ConsumerInactiveThreshold int32 `json:"consumerInactiveThreshold"` @@ -20,21 +27,53 @@ type StreamConfiguration struct { } type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` + ProviderID_ string `json:"providerId"` Subjects []string `json:"subjects"` StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (s *SubscriptionEventConfiguration) ProviderID() string { + return s.ProviderID_ +} + +// ProviderType returns the provider type +func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeNats +} + +// RootFieldName returns the root field name +func (s *SubscriptionEventConfiguration) RootFieldName() string { + return s.RootFieldName_ } type PublishAndRequestEventConfiguration struct { - ProviderID string `json:"providerId"` - Subject string `json:"subject"` - Data json.RawMessage `json:"data"` + ProviderID_ string `json:"providerId"` + Subject string `json:"subject"` + Event Event `json:"event"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (p *PublishAndRequestEventConfiguration) ProviderID() string { + return p.ProviderID_ +} + +// ProviderType returns the provider type +func (p *PublishAndRequestEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeNats +} + +// RootFieldName returns the root field name +func (p *PublishAndRequestEventConfiguration) RootFieldName() string { + return p.RootFieldName_ } func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "data": %s, "providerId":"%s"}`, s.Subject, s.Data, s.ProviderID) + return fmt.Sprintf(`{"subject":"%s", "data": %s, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID_) } type SubscriptionSource struct { @@ -112,3 +151,8 @@ func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *byt func (s *NatsRequestDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) error { panic("not implemented") } + +// Interface compliance checks +var _ datasource.SubscriptionEventConfiguration = (*SubscriptionEventConfiguration)(nil) +var _ datasource.PublishEventConfiguration = (*PublishAndRequestEventConfiguration)(nil) +var _ datasource.StreamEvent = (*Event)(nil) diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 48fd2849f7..62d7005d49 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -63,9 +63,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri subject := c.subjects[0] evtCfg := PublishAndRequestEventConfiguration{ - ProviderID: c.providerId, - Subject: subject, - Data: eventData, + ProviderID_: c.providerId, + Subject: subject, + Event: Event{Data: eventData}, + RootFieldName_: c.fieldName, } return evtCfg.MarshalJSONTemplate(), nil @@ -79,8 +80,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID: c.providerId, - Subjects: c.subjects, + ProviderID_: c.providerId, + Subjects: c.subjects, + RootFieldName_: c.fieldName, } if c.withStreamConfiguration { evtCfg.StreamConfiguration = &StreamConfiguration{ diff --git a/router/pkg/pubsub/nats/engine_datasource_factory_test.go b/router/pkg/pubsub/nats/engine_datasource_factory_test.go index 57426ad34c..50e20a98e7 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory_test.go @@ -34,7 +34,7 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // Configure mock expectations for Publish mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { - return event.ProviderID == "test-provider" && event.Subject == "test-subject" + return event.ProviderID() == "test-provider" && event.Subject == "test-subject" })).Return(nil) // Create the data source with mock adapter @@ -167,7 +167,7 @@ func TestEngineDataSourceFactory_RequestDataSource(t *testing.T) { // Configure mock expectations for Request mockAdapter.On("Request", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { - return event.ProviderID == "test-provider" && event.Subject == "test-subject" + return event.ProviderID() == "test-provider" && event.Subject == "test-subject" }), mock.Anything).Return(nil).Run(func(args mock.Arguments) { w := args.Get(2).(io.Writer) w.Write([]byte(`{"response": "test"}`)) diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index da21d4de88..767e505c65 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -25,18 +25,18 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishAndRequestEventConfiguration{ - ProviderID: "test-provider", - Subject: "test-subject", - Data: json.RawMessage(`{"message":"hello"}`), + ProviderID_: "test-provider", + Subject: "test-subject", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"subject":"test-subject", "data": {"message":"hello"}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishAndRequestEventConfiguration{ - ProviderID: "test-provider-id", - Subject: "subject-with-hyphens", - Data: json.RawMessage(`{"message":"special \"quotes\" here"}`), + ProviderID_: "test-provider-id", + Subject: "subject-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"subject":"subject-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, }, @@ -59,18 +59,18 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishAndRequestEventConfiguration{ - ProviderID: "test-provider", - Subject: "test-subject", - Data: json.RawMessage(`{"message":"hello"}`), + ProviderID_: "test-provider", + Subject: "test-subject", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"subject":"test-subject", "data": {"message":"hello"}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishAndRequestEventConfiguration{ - ProviderID: "test-provider-id", - Subject: "subject-with-hyphens", - Data: json.RawMessage(`{"message":"special \"quotes\" here"}`), + ProviderID_: "test-provider-id", + Subject: "subject-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"subject":"subject-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, }, @@ -148,8 +148,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Subjects: []string{"subject1", "subject2"}, + ProviderID_: "test-provider", + Subjects: []string{"subject1", "subject2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -159,8 +159,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"subjects":["subject1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Subjects: []string{"subject1"}, + ProviderID_: "test-provider", + Subjects: []string{"subject1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, @@ -213,12 +213,12 @@ func TestNatsPublishDataSource_Load(t *testing.T) { }{ { name: "successful publish", - input: `{"subject":"test-subject", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { - return event.ProviderID == "test-provider" && + return event.ProviderID() == "test-provider" && event.Subject == "test-subject" && - string(event.Data) == `{"message":"hello"}` + string(event.Event.Data) == `{"message":"hello"}` })).Return(nil) }, expectError: false, @@ -227,7 +227,7 @@ func TestNatsPublishDataSource_Load(t *testing.T) { }, { name: "publish error", - input: `{"subject":"test-subject", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) }, @@ -288,12 +288,12 @@ func TestNatsRequestDataSource_Load(t *testing.T) { }{ { name: "successful request", - input: `{"subject":"test-subject", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Request", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { - return event.ProviderID == "test-provider" && + return event.ProviderID() == "test-provider" && event.Subject == "test-subject" && - string(event.Data) == `{"message":"hello"}` + string(event.Event.Data) == `{"message":"hello"}` }), mock.Anything).Run(func(args mock.Arguments) { // Write response to the output buffer w := args.Get(2).(io.Writer) @@ -305,7 +305,7 @@ func TestNatsRequestDataSource_Load(t *testing.T) { }, { name: "request error", - input: `{"subject":"test-subject", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Request", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("request error")) }, diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 3efcabbf92..9c99f4f173 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -76,7 +76,7 @@ func (p *ProviderAdapter) Shutdown(ctx context.Context) error { func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "subscribe"), zap.Strings("channels", event.Channels), ) @@ -127,14 +127,14 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { log := p.logger.With( - zap.String("provider_id", event.ProviderID), + zap.String("provider_id", event.ProviderID()), zap.String("method", "publish"), zap.String("channel", event.Channel), ) - log.Debug("publish", zap.ByteString("data", event.Data)) + log.Debug("publish", zap.ByteString("data", event.Event.Data)) - data, dataErr := event.Data.MarshalJSON() + data, dataErr := event.Event.Data.MarshalJSON() if dataErr != nil { log.Error("error marshalling data", zap.Error(dataErr)) return datasource.NewError("error marshalling data", dataErr) diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index d24a4fb959..9c10409088 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -9,25 +9,63 @@ import ( "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) +// Event represents an event from Redis +type Event struct { + Data json.RawMessage `json:"data"` +} + // SubscriptionEventConfiguration contains configuration for subscription events type SubscriptionEventConfiguration struct { - ProviderID string `json:"providerId"` - Channels []string `json:"channels"` + ProviderID_ string `json:"providerId"` + Channels []string `json:"channels"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (s *SubscriptionEventConfiguration) ProviderID() string { + return s.ProviderID_ +} + +// ProviderType returns the provider type +func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeRedis +} + +// RootFieldName returns the root field name +func (s *SubscriptionEventConfiguration) RootFieldName() string { + return s.RootFieldName_ } // PublishEventConfiguration contains configuration for publish events type PublishEventConfiguration struct { - ProviderID string `json:"providerId"` - Channel string `json:"channel"` - Data json.RawMessage `json:"data"` + ProviderID_ string `json:"providerId"` + Channel string `json:"channel"` + Event Event `json:"event"` + RootFieldName_ string `json:"rootFieldName"` +} + +// ProviderID returns the provider ID +func (p *PublishEventConfiguration) ProviderID() string { + return p.ProviderID_ +} + +// ProviderType returns the provider type +func (p *PublishEventConfiguration) ProviderType() datasource.ProviderType { + return datasource.ProviderTypeRedis +} + +// RootFieldName returns the root field name +func (p *PublishEventConfiguration) RootFieldName() string { + return p.RootFieldName_ } func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { - return fmt.Sprintf(`{"channel":"%s", "data": %s, "providerId":"%s"}`, s.Channel, s.Data, s.ProviderID), nil + return fmt.Sprintf(`{"channel":"%s", "data": %s, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID_), nil } // SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis @@ -97,3 +135,8 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B func (s *PublishDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) (err error) { panic("not implemented") } + +// Interface compliance checks +var _ datasource.SubscriptionEventConfiguration = (*SubscriptionEventConfiguration)(nil) +var _ datasource.PublishEventConfiguration = (*PublishEventConfiguration)(nil) +var _ datasource.StreamEvent = (*Event)(nil) diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index c5383ff16a..d2a1d1d0d4 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -59,9 +59,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri providerId := c.providerId evtCfg := PublishEventConfiguration{ - ProviderID: providerId, - Channel: channel, - Data: eventData, + ProviderID_: providerId, + Channel: channel, + Event: Event{Data: eventData}, + RootFieldName_: c.fieldName, } return evtCfg.MarshalJSONTemplate() @@ -77,8 +78,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc // ResolveDataSourceSubscriptionInput builds the input for the subscription data source func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID: c.providerId, - Channels: c.channels, + ProviderID_: c.providerId, + Channels: c.channels, + RootFieldName_: c.fieldName, } object, err := json.Marshal(evtCfg) if err != nil { diff --git a/router/pkg/pubsub/redis/engine_datasource_factory_test.go b/router/pkg/pubsub/redis/engine_datasource_factory_test.go index 0c1344048a..3d7910cf23 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory_test.go @@ -33,7 +33,7 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // Configure mock expectations for Publish mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { - return event.ProviderID == "test-provider" && event.Channel == "test-channel" + return event.ProviderID() == "test-provider" && event.Channel == "test-channel" })).Return(nil) // Create the data source with mock adapter diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 7c47d47cc6..2fd8b87123 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -24,18 +24,18 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishEventConfiguration{ - ProviderID: "test-provider", - Channel: "test-channel", - Data: json.RawMessage(`{"message":"hello"}`), + ProviderID_: "test-provider", + Channel: "test-channel", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"channel":"test-channel", "data": {"message":"hello"}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishEventConfiguration{ - ProviderID: "test-provider-id", - Channel: "channel-with-hyphens", - Data: json.RawMessage(`{"message":"special \"quotes\" here"}`), + ProviderID_: "test-provider-id", + Channel: "channel-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"channel":"channel-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, }, @@ -114,8 +114,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Channels: []string{"channel1", "channel2"}, + ProviderID_: "test-provider", + Channels: []string{"channel1", "channel2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -125,8 +125,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"channels":["channel1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID: "test-provider", - Channels: []string{"channel1"}, + ProviderID_: "test-provider", + Channels: []string{"channel1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, @@ -179,12 +179,12 @@ func TestRedisPublishDataSource_Load(t *testing.T) { }{ { name: "successful publish", - input: `{"channel":"test-channel", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { - return event.ProviderID == "test-provider" && + return event.ProviderID() == "test-provider" && event.Channel == "test-channel" && - string(event.Data) == `{"message":"hello"}` + string(event.Event.Data) == `{"message":"hello"}` })).Return(nil) }, expectError: false, @@ -193,7 +193,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { }, { name: "publish error", - input: `{"channel":"test-channel", "data":{"message":"hello"}, "providerId":"test-provider"}`, + input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) }, From 7ececf08b946596d6455f5fa3df5d2c18eab2090 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 17:34:59 +0200 Subject: [PATCH 040/173] chore: remove local file --- router-tests/.tool-versions | 1 - 1 file changed, 1 deletion(-) delete mode 100644 router-tests/.tool-versions diff --git a/router-tests/.tool-versions b/router-tests/.tool-versions deleted file mode 100644 index 99741b0b8c..0000000000 --- a/router-tests/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.23.9 From 1ec481674d20180ea16b0c19e21f4d2e0348f9d2 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 17:42:15 +0200 Subject: [PATCH 041/173] chore: use ProviderID() --- router/pkg/pubsub/kafka/engine_datasource.go | 2 +- router/pkg/pubsub/nats/engine_datasource.go | 2 +- router/pkg/pubsub/redis/engine_datasource.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 62a83aadd9..06e06477f4 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -67,7 +67,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { func (s *PublishEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"topic":"%s", "data": %s, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID_) + return fmt.Sprintf(`{"topic":"%s", "data": %s, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID()) } type SubscriptionDataSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index a147c2f08f..988435d19b 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -73,7 +73,7 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "data": %s, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID_) + return fmt.Sprintf(`{"subject":"%s", "data": %s, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()) } type SubscriptionSource struct { diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 9c10409088..119f6a46be 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -65,7 +65,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { } func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { - return fmt.Sprintf(`{"channel":"%s", "data": %s, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID_), nil + return fmt.Sprintf(`{"channel":"%s", "data": %s, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID()), nil } // SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis From b938232c3db09f615c45f4f0541d0cc6f582a495 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 17:52:49 +0200 Subject: [PATCH 042/173] chore: update demo to use no Event structure --- demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go | 4 ++-- demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go index c468cc3147..6dd313493e 100644 --- a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go @@ -18,7 +18,7 @@ func (r *mutationResolver) UpdateAvailability(ctx context.Context, employeeID in storage.Set(employeeID, isAvailable) err := r.NatsPubSubByProviderID["default"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdated.%d", employeeID)), - Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID)), + Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, }) if err != nil { @@ -26,7 +26,7 @@ func (r *mutationResolver) UpdateAvailability(ctx context.Context, employeeID in } err = r.NatsPubSubByProviderID["my-nats"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdatedMyNats.%d", employeeID)), - Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID)), + Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, }) if err != nil { diff --git a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go index 2f8ea33149..82a0a7e9f2 100644 --- a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go @@ -21,7 +21,7 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood if r.NatsPubSubByProviderID["default"] != nil { err := r.NatsPubSubByProviderID["default"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ Subject: myNatsTopic, - Data: []byte(payload), + Event: nats.Event{Data: []byte(payload)}, }) if err != nil { return nil, err @@ -34,7 +34,7 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood if r.NatsPubSubByProviderID["my-nats"] != nil { err := r.NatsPubSubByProviderID["my-nats"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ Subject: defaultTopic, - Data: []byte(payload), + Event: nats.Event{Data: []byte(payload)}, }) if err != nil { return nil, err From f69b4cd591c3e9cfbf98eb03f3d55311b3bc7c9d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 18:25:01 +0200 Subject: [PATCH 043/173] fix: align event configuration format --- router/pkg/pubsub/kafka/engine_datasource.go | 2 +- router/pkg/pubsub/nats/engine_datasource.go | 2 +- router/pkg/pubsub/redis/engine_datasource.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 06e06477f4..fff32fcaa5 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -67,7 +67,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { func (s *PublishEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"topic":"%s", "data": %s, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID()) + return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID()) } type SubscriptionDataSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 988435d19b..c100dcf1e7 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -73,7 +73,7 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() string { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "data": %s, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()) + return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()) } type SubscriptionSource struct { diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 119f6a46be..7020197e56 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -65,7 +65,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { } func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { - return fmt.Sprintf(`{"channel":"%s", "data": %s, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID()), nil + return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID()), nil } // SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis From 74f32fca8c8cd8130b44bf1b7edfa07b45ef77ad Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 18:27:55 +0200 Subject: [PATCH 044/173] fix: update MarshalJSONTemplate for new internal event format --- router/pkg/pubsub/kafka/engine_datasource_test.go | 4 ++-- router/pkg/pubsub/nats/engine_datasource_test.go | 8 ++++---- router/pkg/pubsub/redis/engine_datasource_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 854db703f9..2989d5a3cb 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -28,7 +28,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Topic: "test-topic", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, - wantPattern: `{"topic":"test-topic", "data": {"message":"hello"}, "providerId":"test-provider"}`, + wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", @@ -37,7 +37,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Topic: "topic-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, - wantPattern: `{"topic":"topic-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, } diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 767e505c65..68b1f5aa4b 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -29,7 +29,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Subject: "test-subject", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, - wantPattern: `{"subject":"test-subject", "data": {"message":"hello"}, "providerId":"test-provider"}`, + wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", @@ -38,7 +38,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Subject: "subject-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, - wantPattern: `{"subject":"subject-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, + wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, } @@ -63,7 +63,7 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { Subject: "test-subject", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, - wantPattern: `{"subject":"test-subject", "data": {"message":"hello"}, "providerId":"test-provider"}`, + wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", @@ -72,7 +72,7 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { Subject: "subject-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, - wantPattern: `{"subject":"subject-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, + wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, } diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 2fd8b87123..81b6b2b88b 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -28,7 +28,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Channel: "test-channel", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, - wantPattern: `{"channel":"test-channel", "data": {"message":"hello"}, "providerId":"test-provider"}`, + wantPattern: `{"channel":"test-channel", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", @@ -37,7 +37,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Channel: "channel-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, - wantPattern: `{"channel":"channel-with-hyphens", "data": {"message":"special \"quotes\" here"}, "providerId":"test-provider-id"}`, + wantPattern: `{"channel":"channel-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, } From 1649ef5e59fbe87472a96617678d1ebbe83311c7 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 18:29:57 +0200 Subject: [PATCH 045/173] chore: remove duplicated test --- .../pkg/pubsub/nats/engine_datasource_test.go | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 68b1f5aa4b..edc957fbac 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -16,40 +16,6 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { - tests := []struct { - name string - config PublishAndRequestEventConfiguration - wantPattern string - }{ - { - name: "simple configuration", - config: PublishAndRequestEventConfiguration{ - ProviderID_: "test-provider", - Subject: "test-subject", - Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, - }, - wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, - }, - { - name: "with special characters", - config: PublishAndRequestEventConfiguration{ - ProviderID_: "test-provider-id", - Subject: "subject-with-hyphens", - Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, - }, - wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.config.MarshalJSONTemplate() - assert.Equal(t, tt.wantPattern, result) - }) - } -} - func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { tests := []struct { name string From b83e4ef980230ca6a5b94f28bdd4b2e301f407a6 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 15 Jul 2025 18:37:11 +0200 Subject: [PATCH 046/173] chore: improve naming --- router/pkg/pubsub/kafka/engine_datasource.go | 22 +++++++++---------- .../pubsub/kafka/engine_datasource_factory.go | 14 ++++++------ .../pubsub/kafka/engine_datasource_test.go | 20 ++++++++--------- router/pkg/pubsub/nats/engine_datasource.go | 22 +++++++++---------- .../pubsub/nats/engine_datasource_factory.go | 14 ++++++------ .../pkg/pubsub/nats/engine_datasource_test.go | 20 ++++++++--------- router/pkg/pubsub/redis/engine_datasource.go | 22 +++++++++---------- .../pubsub/redis/engine_datasource_factory.go | 14 ++++++------ .../pubsub/redis/engine_datasource_test.go | 20 ++++++++--------- 9 files changed, 84 insertions(+), 84 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index fff32fcaa5..2b29ba9899 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -22,14 +22,14 @@ type Event struct { } type SubscriptionEventConfiguration struct { - ProviderID_ string `json:"providerId"` - Topics []string `json:"topics"` - RootFieldName_ string `json:"rootFieldName"` + Provider string `json:"providerId"` + Topics []string `json:"topics"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (s *SubscriptionEventConfiguration) ProviderID() string { - return s.ProviderID_ + return s.Provider } // ProviderType returns the provider type @@ -39,19 +39,19 @@ func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType // RootFieldName returns the root field name func (s *SubscriptionEventConfiguration) RootFieldName() string { - return s.RootFieldName_ + return s.FieldName } type PublishEventConfiguration struct { - ProviderID_ string `json:"providerId"` - Topic string `json:"topic"` - Event Event `json:"event"` - RootFieldName_ string `json:"rootFieldName"` + Provider string `json:"providerId"` + Topic string `json:"topic"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (p *PublishEventConfiguration) ProviderID() string { - return p.ProviderID_ + return p.Provider } // ProviderType returns the provider type @@ -61,7 +61,7 @@ func (p *PublishEventConfiguration) ProviderType() datasource.ProviderType { // RootFieldName returns the root field name func (p *PublishEventConfiguration) RootFieldName() string { - return p.RootFieldName_ + return p.FieldName } func (s *PublishEventConfiguration) MarshalJSONTemplate() string { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index e46250b4a8..95589d6f9e 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -49,10 +49,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri } evtCfg := PublishEventConfiguration{ - ProviderID_: c.providerId, - Topic: c.topics[0], - Event: Event{Data: eventData}, - RootFieldName_: c.fieldName, + Provider: c.providerId, + Topic: c.topics[0], + Event: Event{Data: eventData}, + FieldName: c.fieldName, } return evtCfg.MarshalJSONTemplate(), nil @@ -66,9 +66,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID_: c.providerId, - Topics: c.topics, - RootFieldName_: c.fieldName, + Provider: c.providerId, + Topics: c.topics, + FieldName: c.fieldName, } object, err := json.Marshal(evtCfg) if err != nil { diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 2989d5a3cb..650153f3d1 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -24,18 +24,18 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishEventConfiguration{ - ProviderID_: "test-provider", - Topic: "test-topic", - Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + Provider: "test-provider", + Topic: "test-topic", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishEventConfiguration{ - ProviderID_: "test-provider-id", - Topic: "topic-with-hyphens", - Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + Provider: "test-provider-id", + Topic: "topic-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, @@ -113,8 +113,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Topics: []string{"topic1", "topic2"}, + Provider: "test-provider", + Topics: []string{"topic1", "topic2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -124,8 +124,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"topics":["topic1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Topics: []string{"topic1"}, + Provider: "test-provider", + Topics: []string{"topic1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index c100dcf1e7..5e5227028f 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -14,7 +14,7 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -// NatsEvent represents an event from NATS +// Event represents an event from NATS type Event struct { Data json.RawMessage `json:"data"` Metadata map[string]string `json:"metadata"` @@ -27,15 +27,15 @@ type StreamConfiguration struct { } type SubscriptionEventConfiguration struct { - ProviderID_ string `json:"providerId"` + Provider string `json:"providerId"` Subjects []string `json:"subjects"` StreamConfiguration *StreamConfiguration `json:"streamConfiguration,omitempty"` - RootFieldName_ string `json:"rootFieldName"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (s *SubscriptionEventConfiguration) ProviderID() string { - return s.ProviderID_ + return s.Provider } // ProviderType returns the provider type @@ -45,19 +45,19 @@ func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType // RootFieldName returns the root field name func (s *SubscriptionEventConfiguration) RootFieldName() string { - return s.RootFieldName_ + return s.FieldName } type PublishAndRequestEventConfiguration struct { - ProviderID_ string `json:"providerId"` - Subject string `json:"subject"` - Event Event `json:"event"` - RootFieldName_ string `json:"rootFieldName"` + Provider string `json:"providerId"` + Subject string `json:"subject"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (p *PublishAndRequestEventConfiguration) ProviderID() string { - return p.ProviderID_ + return p.Provider } // ProviderType returns the provider type @@ -67,7 +67,7 @@ func (p *PublishAndRequestEventConfiguration) ProviderType() datasource.Provider // RootFieldName returns the root field name func (p *PublishAndRequestEventConfiguration) RootFieldName() string { - return p.RootFieldName_ + return p.FieldName } func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() string { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 62d7005d49..a8aa031fbc 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -63,10 +63,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri subject := c.subjects[0] evtCfg := PublishAndRequestEventConfiguration{ - ProviderID_: c.providerId, - Subject: subject, - Event: Event{Data: eventData}, - RootFieldName_: c.fieldName, + Provider: c.providerId, + Subject: subject, + Event: Event{Data: eventData}, + FieldName: c.fieldName, } return evtCfg.MarshalJSONTemplate(), nil @@ -80,9 +80,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID_: c.providerId, - Subjects: c.subjects, - RootFieldName_: c.fieldName, + Provider: c.providerId, + Subjects: c.subjects, + FieldName: c.fieldName, } if c.withStreamConfiguration { evtCfg.StreamConfiguration = &StreamConfiguration{ diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index edc957fbac..b243e27536 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -25,18 +25,18 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishAndRequestEventConfiguration{ - ProviderID_: "test-provider", - Subject: "test-subject", - Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + Provider: "test-provider", + Subject: "test-subject", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishAndRequestEventConfiguration{ - ProviderID_: "test-provider-id", - Subject: "subject-with-hyphens", - Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + Provider: "test-provider-id", + Subject: "subject-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, @@ -114,8 +114,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Subjects: []string{"subject1", "subject2"}, + Provider: "test-provider", + Subjects: []string{"subject1", "subject2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -125,8 +125,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"subjects":["subject1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Subjects: []string{"subject1"}, + Provider: "test-provider", + Subjects: []string{"subject1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 7020197e56..72ab322d88 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -21,14 +21,14 @@ type Event struct { // SubscriptionEventConfiguration contains configuration for subscription events type SubscriptionEventConfiguration struct { - ProviderID_ string `json:"providerId"` - Channels []string `json:"channels"` - RootFieldName_ string `json:"rootFieldName"` + Provider string `json:"providerId"` + Channels []string `json:"channels"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (s *SubscriptionEventConfiguration) ProviderID() string { - return s.ProviderID_ + return s.Provider } // ProviderType returns the provider type @@ -38,20 +38,20 @@ func (s *SubscriptionEventConfiguration) ProviderType() datasource.ProviderType // RootFieldName returns the root field name func (s *SubscriptionEventConfiguration) RootFieldName() string { - return s.RootFieldName_ + return s.FieldName } // PublishEventConfiguration contains configuration for publish events type PublishEventConfiguration struct { - ProviderID_ string `json:"providerId"` - Channel string `json:"channel"` - Event Event `json:"event"` - RootFieldName_ string `json:"rootFieldName"` + Provider string `json:"providerId"` + Channel string `json:"channel"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` } // ProviderID returns the provider ID func (p *PublishEventConfiguration) ProviderID() string { - return p.ProviderID_ + return p.Provider } // ProviderType returns the provider type @@ -61,7 +61,7 @@ func (p *PublishEventConfiguration) ProviderType() datasource.ProviderType { // RootFieldName returns the root field name func (p *PublishEventConfiguration) RootFieldName() string { - return p.RootFieldName_ + return p.FieldName } func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index d2a1d1d0d4..0c62bed033 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -59,10 +59,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri providerId := c.providerId evtCfg := PublishEventConfiguration{ - ProviderID_: providerId, - Channel: channel, - Event: Event{Data: eventData}, - RootFieldName_: c.fieldName, + Provider: providerId, + Channel: channel, + Event: Event{Data: eventData}, + FieldName: c.fieldName, } return evtCfg.MarshalJSONTemplate() @@ -78,9 +78,9 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.Subsc // ResolveDataSourceSubscriptionInput builds the input for the subscription data source func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { evtCfg := SubscriptionEventConfiguration{ - ProviderID_: c.providerId, - Channels: c.channels, - RootFieldName_: c.fieldName, + Provider: c.providerId, + Channels: c.channels, + FieldName: c.fieldName, } object, err := json.Marshal(evtCfg) if err != nil { diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 81b6b2b88b..e20d326aba 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -24,18 +24,18 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { { name: "simple configuration", config: PublishEventConfiguration{ - ProviderID_: "test-provider", - Channel: "test-channel", - Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + Provider: "test-provider", + Channel: "test-channel", + Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, wantPattern: `{"channel":"test-channel", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, }, { name: "with special characters", config: PublishEventConfiguration{ - ProviderID_: "test-provider-id", - Channel: "channel-with-hyphens", - Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + Provider: "test-provider-id", + Channel: "channel-with-hyphens", + Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, wantPattern: `{"channel":"channel-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, }, @@ -114,8 +114,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Channels: []string{"channel1", "channel2"}, + Provider: "test-provider", + Channels: []string{"channel1", "channel2"}, }, mock.Anything).Return(nil) }, expectError: false, @@ -125,8 +125,8 @@ func TestSubscriptionSource_Start(t *testing.T) { input: `{"channels":["channel1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - ProviderID_: "test-provider", - Channels: []string{"channel1"}, + Provider: "test-provider", + Channels: []string{"channel1"}, }, mock.Anything).Return(errors.New("subscription error")) }, expectError: true, From 9ed94fd5590a648f33df7f0b377ba0821594130e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 16 Jul 2025 10:23:55 +0200 Subject: [PATCH 047/173] chore: make the ADR from the approved RFC --- adr/cosmo-streams-v1.md | 286 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 adr/cosmo-streams-v1.md diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md new file mode 100644 index 0000000000..a1005a0ce8 --- /dev/null +++ b/adr/cosmo-streams-v1.md @@ -0,0 +1,286 @@ +--- +title: "Cosmo Streams v1" +author: Alessandro Pagnin +date: 2025-07-16 +status: Accepted +--- + +# ADR - Cosmo Streams V1 + +- **Author:** Alessandro Pagnin +- **Date:** 2025-07-16 +- **Status:** Accepted +- **RFC:** ../rfcs/cosmo-streams-v1.md + +## Abstract +This ADR describes new hooks that will be added to the router to support more customizable stream behavior. +The goal is to allow developers to customize the cosmo streams behavior. + +## Decision +A developer can implement a custom module by creating a struct that implements the following interfaces: + +- `SubscriptionOnStartHandler`: Called once at subscription start. +- `StreamBatchEventHook`: Called each time a batch of events is received from the provider. +- `StreamPublishEventHook`: Called each time a batch of events is going to be sent to the provider. + +```go +// STRUCTURES TO BE ADDED TO PUBSUB PACKAGE +type ProviderType string +const ( + ProviderTypeNats ProviderType = "nats" + ProviderTypeKafka ProviderType = "kafka" + ProviderTypeRedis ProviderType = "redis" +} + +// StreamHookError is used to customize the error messages and the behavior +type StreamHookError struct { + HttpError core.HttpError + CloseSubscription bool +} + +// OperationContext already exists, we just have to add the Variables() method +type OperationContext interface { + Name() string + // the variables are currently not available, so we need to expose them here + Variables() *astjson.Value +} + +// each provider will have its own event type with custom fields +// the StreamEvent interface is used to allow the hooks system to be provider-agnostic +// there could be common fields in future, but for now we don't need them +type StreamEvent interface {} + +// SubscriptionEventConfiguration is the common interface for the subscription event configuration +type SubscriptionEventConfiguration interface { + ProviderID() string + ProviderType() string + // the root field name of the subscription in the schema + RootFieldName() string +} + +// PublishEventConfiguration is the common interface for the publish event configuration +type PublishEventConfiguration interface { + ProviderID() string + ProviderType() string + // the root field name of the mutation in the schema + RootFieldName() string +} + +type SubscriptionOnStartHookContext interface { + // the request context + RequestContext() RequestContext + // the stream context + StreamContext() StreamContext + // the subscription event configuration + SubscriptionEventConfiguration() SubscriptionEventConfiguration + // write an event to the stream of the current subscription + WriteEvent(event core.StreamEvent) +} + +type SubscriptionOnStartHandler interface { + // OnSubscriptionOnStart is called once at subscription start + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error +} + +type StreamBatchEventHookContext interface { + // the request context + RequestContext() RequestContext + // the subscription event configuration + SubscriptionEventConfiguration() SubscriptionEventConfiguration +} + +type StreamBatchEventHook interface { + // OnStreamEvents is called each time a batch of events is received from the provider + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} + +type StreamPublishEventHookContext interface { + // the request context + RequestContext() RequestContext + // the publish event configuration + PublishEventConfiguration() PublishEventConfiguration +} + +type StreamPublishEventHook interface { + // OnPublishEvents is called each time a batch of events is going to be sent to the provider + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + OnPublishEvents(ctx StreamPublishEventHookContext, events []StreamEvent) ([]StreamEvent, error) +} +``` + +## Example Use Cases + +- **Authorization**: Implementing authorization checks at the start of subscriptions +- **Initial message**: Sending an initial message to clients upon subscription start +- **Data mapping**: Transforming events data from the format that could be used by the external system to/from Federation compatible Router events +- **Event filtering**: Filtering events using custom logic + +## Backwards Compatibility + +The new hooks can be integrated in the router in a fully backwards compatible way. +When the new module system will be released, some changes will be needed. + +# Example Modules + +__All examples are pseudocode and not tested, but they are as close as possible to the final implementation__ + +## Filter and remap events + +This example will show how to filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. + +### 1. Add a subscription to the cosmo streams graphql schema + +The developer will start by adding a subscription to the cosmo streams graphql schema. + +```graphql +type Subscription { + employeeUpdates(): Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") +} + +type Employee @key(fields: "id", resolvable: false) { + id: Int! @external +} +``` +After publishing the schema, the developer will need to add the module to the cosmo streams engine. + +### 2. Write the custom module + +The developer will need to write the custom module that will be used to subscribe to the `employeeUpdates` subject and filter the events based on the client's scopes and remapping the messages as they are expected from the `Employee` type. + +```go +package mymodule + +import ( + "encoding/json" + "slices" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" +) + +func init() { + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) +} + +type MyModule struct {} + +func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { + // check if the provider is nats + if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { + return events, nil + } + + // check if the provider id is the one expected by the module + if ctx.StreamContext().ProviderID() != "my-nats" { + return events, nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "employeeUpdates" { + return events, nil + } + + // check if the client is authenticated + if ctx.RequestContext().Authentication() == nil { + // if the client is not authenticated, return no events + return events, nil + } + + // check if the client is allowed to subscribe to the stream + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + if !found { + return events, fmt.Errorf("client is not allowed to subscribe to the stream") + } + + newEvents := make([]core.StreamEvent, 0, len(events)) + + for _, evt := range events { + natsEvent, ok := evt.(*nats.NatsEvent); + if !ok { + newEvents = append(newEvents, evt) + continue + } + + // decode the event data coming from the provider + var dataReceived struct { + EmployeeId string `json:"EmployeeId"` + OtherField string `json:"OtherField"` + } + err := json.Unmarshal(natsEvent.Data, &dataReceived) + if err != nil { + return events, fmt.Errorf("error unmarshalling data: %w", err) + } + + // filter the events based on the client's scopes + if !slices.Contains(clientAllowedEntitiesIds, dataReceived.EmployeeId) { + continue + } + + // prepare the data to send to the client + var dataToSend struct { + Id string `json:"id"` + TypeName string `json:"__typename"` + } + dataToSend.Id = dataReceived.EmployeeId + dataToSend.TypeName = "Employee" + + // marshal the data to send to the client + dataToSendMarshalled, err := json.Marshal(dataToSend) + if err != nil { + return events, fmt.Errorf("error marshalling data: %w", err) + } + + // create the new event + newEvent := &nats.NatsEvent{ + Data: dataToSendMarshalled, + Metadata: natsEvent.Metadata, + } + newEvents = append(newEvents, newEvent) + } + return newEvents, nil +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} + +// Interface guards +var ( + _ core.StreamBatchEventHook = (*MyModule)(nil) +) +``` + +### 3. Add the provider configuration to the cosmo router +```yaml +version: "1" + +events: + providers: + nats: + - id: my-nats + url: "nats://localhost:4222" +``` + +### 4. Build the cosmo router with the custom module + +Build and run the router with the custom module added. + +# Outlook + +## Using AsyncAPI for Event Data Structure + +We could use AsyncAPI specifications to define the event data structure and generate the Go structs automatically. This would make the development of custom modules easier and more maintainable. +We could also generate the AsyncAPI specification from the schema and the events data, to make it easier for external systems to use the events published by cosmo streams engine. + +## Generate hooks from AsyncAPI specifications + +Building on the AsyncAPI integration, we could allow the user to define their streams using AsyncAPI and generate fully typesafe hooks with all events structures generated from the AsyncAPI specification. \ No newline at end of file From 62a0882bc6474a1f993707fd7cf0694216c7a393 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 16 Jul 2025 11:05:15 +0200 Subject: [PATCH 048/173] chore: add an example to adr --- adr/cosmo-streams-v1.md | 111 ++++++++++++++++++++++++++++++++++++++++ rfc/cosmo-streams-v1.md | 8 ++- 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index a1005a0ce8..f02729dfa6 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -270,6 +270,117 @@ events: url: "nats://localhost:4222" ``` +## Check authorization at subscription start + +This example will show how to check the authorization at subscription start. + +### 1. Add a subscription to the cosmo streams graphql schema + +The developer will start by adding a subscription to the cosmo streams graphql schema. + +```graphql +type Subscription { + employeeUpdates(): Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") +} + +type Employee @key(fields: "id", resolvable: false) { + id: Int! @external +} +``` +After publishing the schema, the developer will need to add the module to the cosmo streams engine. + +### 2. Write the custom module + +The developer will need to write the custom module that will be used to check the authorization at subscription start. + +```go +package mymodule + +import ( + "encoding/json" + "slices" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" +) + +func init() { + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) +} + +type MyModule struct {} + +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { + // check if the provider is nats + if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { + return nil + } + + // check if the provider id is the one expected by the module + if ctx.StreamContext().ProviderID() != "my-nats" { + return nil + } + + // check if the subject is the one expected by the module + natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) + if natsConfig.Subjects[0] != "employeeUpdates" { + return nil + } + + // check if the client is authenticated + if ctx.RequestContext().Authentication() == nil { + // if the client is not authenticated, return an error + return &StreamHookError{ + HttpError: core.HttpError{ + Code: http.StatusUnauthorized, + Message: "client is not authenticated", + }, + CloseSubscription: true, + } + } + + // check if the client is allowed to subscribe to the stream + clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["readEmployee"] + if !found { + return &StreamHookError{ + HttpError: core.HttpError{ + Code: http.StatusForbidden, + Message: "client is not allowed to read employees", + }, + CloseSubscription: true, + } + } + + return nil +} + +func (m *MyModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + ID: myModuleID, + Priority: 1, + New: func() core.Module { + return &MyModule{} + }, + } +} + +// Interface guards +var ( + _ core.StreamBatchEventHook = (*MyModule)(nil) +) +``` + +### 3. Add the provider configuration to the cosmo router +```yaml +version: "1" + +events: + providers: + nats: + - id: my-nats + url: "nats://localhost:4222" +``` + ### 4. Build the cosmo router with the custom module Build and run the router with the custom module added. diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index 9a32ad2e54..dd76c2e29e 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -217,7 +217,7 @@ type NatsEvent struct { type KafkaEvent struct { Key []byte Data json.RawMessage - Headers map[[]byte]][]byte + Headers map[[]byte][]byte } // StreamBatchEventHook processes a batch of inbound stream events @@ -296,7 +296,7 @@ func (m *MyModule) OnStreamEvents( newEvents = append(newEvents, evt) } - return events, nil + return newEvents, nil } func (m *MyModule) Module() core.ModuleInfo { @@ -800,6 +800,10 @@ type StreamHookError struct { CloseSubscription bool } +func (e StreamHookError) Error() string { + return e.HttpError.Message() +} + // STRUCTURES TO BE ADDED TO PUBSUB PACKAGE type ProviderType string const ( From 0e606037fddfb2a6538079c38e6dd41bf3776129 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 17 Jul 2025 12:15:21 +0200 Subject: [PATCH 049/173] feat: add SubscriptionOnStartHandler --- .../modules/start-subscription/module.go | 44 +++++++++++ .../modules/start_subscription_test.go | 79 +++++++++++++++++++ router/core/executor.go | 4 +- router/core/factoryresolver.go | 24 ++++-- router/core/graph_server.go | 1 + router/core/plan_generator.go | 2 +- router/core/router.go | 4 + router/core/router_config.go | 1 + router/core/streams_modules.go | 57 +++++++++++++ router/pkg/pubsub/datasource/factory.go | 12 +-- .../pkg/pubsub/datasource/hookeddatasource.go | 24 ++++++ router/pkg/pubsub/datasource/planner.go | 7 +- router/pkg/pubsub/datasource/provider.go | 5 ++ router/pkg/pubsub/pubsub.go | 15 ++-- router/pkg/pubsub/pubsub_test.go | 14 ++-- 15 files changed, 266 insertions(+), 27 deletions(-) create mode 100644 router-tests/modules/start-subscription/module.go create mode 100644 router-tests/modules/start_subscription_test.go create mode 100644 router/core/streams_modules.go create mode 100644 router/pkg/pubsub/datasource/hookeddatasource.go diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go new file mode 100644 index 0000000000..2a062f02a3 --- /dev/null +++ b/router-tests/modules/start-subscription/module.go @@ -0,0 +1,44 @@ +package start_subscription + +import ( + "go.uber.org/zap" + + "github.com/wundergraph/cosmo/router/core" +) + +const myModuleID = "startSubscriptionModule" + +type StartSubscriptionModule struct { + Logger *zap.Logger +} + +func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { + // Assign the logger to the module for non-request related logging + m.Logger = ctx.Logger + + return nil +} + +func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { + + m.Logger.Info("SubscriptionOnStart Hook has been run") + + return nil +} + +func (m *StartSubscriptionModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + // This is the ID of your module, it must be unique + ID: myModuleID, + // The priority of your module, lower numbers are executed first + Priority: 1, + New: func() core.Module { + return &StartSubscriptionModule{} + }, + } +} + +// Interface guard +var ( + _ core.SubscriptionOnStartHandler = (*StartSubscriptionModule)(nil) +) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go new file mode 100644 index 0000000000..b16f057a74 --- /dev/null +++ b/router-tests/modules/start_subscription_test.go @@ -0,0 +1,79 @@ +package module_test + +import ( + "testing" + "time" + + "github.com/hasura/go-graphql-client" + start_subscription "github.com/wundergraph/cosmo/router-tests/modules/start-subscription" + "go.uber.org/zap/zapcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router-tests/testenv" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/config" +) + +func TestStartSubscriptionHook(t *testing.T) { + t.Parallel() + + t.Run("Test StartSubscription hook is called", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{}, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + // should never be called + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) +} diff --git a/router/core/executor.go b/router/core/executor.go index 560f40840e..d8ee7145d0 100644 --- a/router/core/executor.go +++ b/router/core/executor.go @@ -35,6 +35,8 @@ type ExecutorConfigurationBuilder struct { subscriptionClientOptions *SubscriptionClientOptions instanceData InstanceData + + startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error } type Executor struct { @@ -214,7 +216,7 @@ func (b *ExecutorConfigurationBuilder) buildPlannerConfiguration(ctx context.Con routerEngineCfg.Execution.EnableSingleFlight, routerEngineCfg.Execution.EnableNetPoll, b.instanceData, - ), b.logger) + ), b.logger, b.startSubscriptionModules) // this generates the plan config using the data source factories from the config package planConfig, providers, err := loader.Load(engineConfig, subgraphs, routerEngineCfg, pluginsEnabled) diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 3753836866..3eb75107ea 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -29,8 +29,9 @@ import ( ) type Loader struct { - ctx context.Context - resolver FactoryResolver + ctx context.Context + resolver FactoryResolver + startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error // includeInfo controls whether additional information like type usage and field usage is included in the plan de includeInfo bool logger *zap.Logger @@ -188,12 +189,13 @@ func (d *DefaultFactoryResolver) InstanceData() InstanceData { return d.instanceData } -func NewLoader(ctx context.Context, includeInfo bool, resolver FactoryResolver, logger *zap.Logger) *Loader { +func NewLoader(ctx context.Context, includeInfo bool, resolver FactoryResolver, logger *zap.Logger, startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error) *Loader { return &Loader{ - ctx: ctx, - resolver: resolver, - includeInfo: includeInfo, - logger: logger, + ctx: ctx, + resolver: resolver, + includeInfo: includeInfo, + logger: logger, + startSubscriptionModules: startSubscriptionModules, } } @@ -467,6 +469,11 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } + onSubscriptionStarts := make([]pubsub_datasource.OnSubscriptionStartFn, len(l.startSubscriptionModules)) + for i, fn := range l.startSubscriptionModules { + onSubscriptionStarts[i] = callSubscriptionOnStart(fn) + } + factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, routerEngineConfig.Events, @@ -474,6 +481,9 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod pubSubDS, l.resolver.InstanceData().HostName, l.resolver.InstanceData().ListenAddress, + pubsub.Hooks{ + OnSubscriptionStarts: onSubscriptionStarts, + }, ) if err != nil { return nil, providers, err diff --git a/router/core/graph_server.go b/router/core/graph_server.go index d112c75552..839e6973a3 100644 --- a/router/core/graph_server.go +++ b/router/core/graph_server.go @@ -1164,6 +1164,7 @@ func (s *graphServer) buildGraphMux( EnableTraceClient: enableTraceClient, CircuitBreaker: s.circuitBreakerManager, }, + startSubscriptionModules: s.startSubscriptionModules, } executor, providers, err := ecb.Build( diff --git a/router/core/plan_generator.go b/router/core/plan_generator.go index 01e45ec915..ba9c8b8072 100644 --- a/router/core/plan_generator.go +++ b/router/core/plan_generator.go @@ -282,7 +282,7 @@ func (pg *PlanGenerator) loadConfiguration(routerConfig *nodev1.RouterConfig, lo httpClient: http.DefaultClient, streamingClient: http.DefaultClient, subscriptionClient: subscriptionClient, - }, logger) + }, logger, nil) // this generates the plan configuration using the data source factories from the config package planConfig, _, err := loader.Load(routerConfig.GetEngineConfig(), routerConfig.GetSubgraphs(), &routerEngineConfig, false) // TODO: configure plugins diff --git a/router/core/router.go b/router/core/router.go index f7aa41de12..bd023849a8 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -642,6 +642,10 @@ func (r *Router) initModules(ctx context.Context) error { } } + if handler, ok := moduleInstance.(SubscriptionOnStartHandler); ok { + r.startSubscriptionModules = append(r.startSubscriptionModules, handler.SubscriptionOnStart) + } + r.modules = append(r.modules, moduleInstance) r.logger.Info("Module registered", diff --git a/router/core/router_config.go b/router/core/router_config.go index 0b70b1a72c..28c4a4481e 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -118,6 +118,7 @@ type Config struct { mcp config.MCPConfiguration plugins config.PluginsConfiguration tracingAttributes []config.CustomAttribute + startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error } // Usage returns an anonymized version of the config for usage tracking diff --git a/router/core/streams_modules.go b/router/core/streams_modules.go new file mode 100644 index 0000000000..264a66942b --- /dev/null +++ b/router/core/streams_modules.go @@ -0,0 +1,57 @@ +package core + +import ( + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +// StreamHookError is used to customize the error messages and the behavior +type StreamHookError struct { + HttpError HttpError + CloseSubscription bool +} + +type SubscriptionOnStartHookContext interface { + // the request context + RequestContext() RequestContext + // the subscription event configuration + SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration + // write an event to the stream of the current subscription + WriteEvent(event datasource.StreamEvent) +} + +type subscriptionOnStartHookContext struct { + requestContext RequestContext + subscriptionEventConfiguration datasource.SubscriptionEventConfiguration +} + +func (c *subscriptionOnStartHookContext) RequestContext() RequestContext { + return c.requestContext +} + +func (c *subscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { + return c.subscriptionEventConfiguration +} + +func (c *subscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { + //c.subscriptionEventConfiguration.WriteEvent(event) +} + +type SubscriptionOnStartHandler interface { + // OnSubscriptionOnStart is called once at subscription start + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error +} + +//write a method that converts from func(ctx SubscriptionOnStartHookContext) error to func(ctx *resolve.Context, event StreamEvent) error + +func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, event datasource.StreamEvent) error { + return func(resolveCtx *resolve.Context, event datasource.StreamEvent) error { + requestContext := getRequestContext(resolveCtx.Context()) + + return fn(&subscriptionOnStartHookContext{ + requestContext: requestContext, + //subscriptionEventConfiguration: subscriptionEventConfiguration, + }) + } +} diff --git a/router/pkg/pubsub/datasource/factory.go b/router/pkg/pubsub/datasource/factory.go index 5221d22dc9..c7b371c244 100644 --- a/router/pkg/pubsub/datasource/factory.go +++ b/router/pkg/pubsub/datasource/factory.go @@ -9,14 +9,16 @@ import ( ) type PlannerConfig[PB ProviderBuilder[P, E], P any, E any] struct { - ProviderBuilder PB - Event E + ProviderBuilder PB + Event E + OnSubscriptionStartFns []OnSubscriptionStartFn } -func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E) *PlannerConfig[PB, P, E] { +func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, onSubscriptionStartFns []OnSubscriptionStartFn) *PlannerConfig[PB, P, E] { return &PlannerConfig[PB, P, E]{ - ProviderBuilder: providerBuilder, - Event: event, + ProviderBuilder: providerBuilder, + Event: event, + OnSubscriptionStartFns: onSubscriptionStartFns, } } diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go new file mode 100644 index 0000000000..23ee76277b --- /dev/null +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -0,0 +1,24 @@ +package datasource + +import ( + "github.com/cespare/xxhash/v2" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +type HookedSubscriptionDataSource struct { + OnSubscriptionStartFns []OnSubscriptionStartFn + SubscriptionDataSource resolve.SubscriptionDataSource +} + +func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { + for _, fn := range h.OnSubscriptionStartFns { + if err := fn(ctx, input); err != nil { + return err + } + } + return h.SubscriptionDataSource.Start(ctx, input, updater) +} + +func (h *HookedSubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) { + return h.SubscriptionDataSource.UniqueRequestID(ctx, input, xxh) +} diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index cde11b6d42..4d2a49d3b2 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -117,6 +117,11 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati return plan.SubscriptionConfiguration{} } + hookedDataSource := &HookedSubscriptionDataSource{ + SubscriptionDataSource: dataSource, + OnSubscriptionStartFns: p.config.OnSubscriptionStartFns, + } + input, err := pubSubDataSource.ResolveDataSourceSubscriptionInput() if err != nil { p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to get resolve data source subscription input: %w", err)) @@ -126,7 +131,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati return plan.SubscriptionConfiguration{ Input: input, Variables: p.variables, - DataSource: dataSource, + DataSource: hookedDataSource, PostProcessing: resolve.PostProcessingConfiguration{ MergePath: []string{pubSubDataSource.GetFieldName()}, }, diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index fb513a7fc7..2aeaaa421d 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -2,6 +2,8 @@ package datasource import ( "context" + + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) type ArgumentTemplateCallback func(tpl string) (string, error) @@ -43,8 +45,11 @@ const ( // StreamEvent is a generic interface for all stream events // Each provider will have its own event type that implements this interface +// there could be common fields in future, but for now we don't need any type StreamEvent interface{} +type OnSubscriptionStartFn func(ctx *resolve.Context, event StreamEvent) error + // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { ProviderID() string diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index c6ec29be82..1afc212662 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -49,6 +49,10 @@ func (e *ProviderNotDefinedError) Error() string { return fmt.Sprintf("%s provider with ID %s is not defined", e.ProviderTypeID, e.ProviderID) } +type Hooks struct { + OnSubscriptionStarts []pubsub_datasource.OnSubscriptionStartFn +} + // BuildProvidersAndDataSources is a generic function that builds providers and data sources for the given // EventsConfiguration and DataSourceConfigurationWithMetadata func BuildProvidersAndDataSources( @@ -58,6 +62,7 @@ func BuildProvidersAndDataSources( dsConfs []DataSourceConfigurationWithMetadata, hostName string, routerListenAddr string, + hooks Hooks, ) ([]pubsub_datasource.Provider, []plan.DataSource, error) { var pubSubProviders []pubsub_datasource.Provider var outs []plan.DataSource @@ -71,7 +76,7 @@ func BuildProvidersAndDataSources( events: dsConf.Configuration.GetCustomEvents().GetKafka(), }) } - kafkaPubSubProviders, kafkaOuts, err := build(ctx, kafkaBuilder, config.Providers.Kafka, kafkaDsConfsWithEvents) + kafkaPubSubProviders, kafkaOuts, err := build(ctx, kafkaBuilder, config.Providers.Kafka, kafkaDsConfsWithEvents, hooks) if err != nil { return nil, nil, err } @@ -87,7 +92,7 @@ func BuildProvidersAndDataSources( events: dsConf.Configuration.GetCustomEvents().GetNats(), }) } - natsPubSubProviders, natsOuts, err := build(ctx, natsBuilder, config.Providers.Nats, natsDsConfsWithEvents) + natsPubSubProviders, natsOuts, err := build(ctx, natsBuilder, config.Providers.Nats, natsDsConfsWithEvents, hooks) if err != nil { return nil, nil, err } @@ -103,7 +108,7 @@ func BuildProvidersAndDataSources( events: dsConf.Configuration.GetCustomEvents().GetRedis(), }) } - redisPubSubProviders, redisOuts, err := build(ctx, redisBuilder, config.Providers.Redis, redisDsConfsWithEvents) + redisPubSubProviders, redisOuts, err := build(ctx, redisBuilder, config.Providers.Redis, redisDsConfsWithEvents, hooks) if err != nil { return nil, nil, err } @@ -113,7 +118,7 @@ func BuildProvidersAndDataSources( return pubSubProviders, outs, nil } -func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E]) ([]pubsub_datasource.Provider, []plan.DataSource, error) { +func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E], hooks Hooks) ([]pubsub_datasource.Provider, []plan.DataSource, error) { var pubSubProviders []pubsub_datasource.Provider var outs []plan.DataSource @@ -154,7 +159,7 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder // build data sources for each event for _, dsConf := range dsConfs { for i, event := range dsConf.events { - plannerConfig := pubsub_datasource.NewPlannerConfig(builder, event) + plannerConfig := pubsub_datasource.NewPlannerConfig(builder, event, hooks.OnSubscriptionStarts) out, err := plan.NewDataSourceConfiguration( dsConf.dsConf.Configuration.Id+"-"+builder.TypeID()+"-"+strconv.Itoa(i), pubsub_datasource.NewPlannerFactory(ctx, plannerConfig), diff --git a/router/pkg/pubsub/pubsub_test.go b/router/pkg/pubsub/pubsub_test.go index a76194f7c5..8085fdd24e 100644 --- a/router/pkg/pubsub/pubsub_test.go +++ b/router/pkg/pubsub/pubsub_test.go @@ -65,7 +65,7 @@ func TestBuild_OK(t *testing.T) { // ctx, kafkaBuilder, config.Providers.Kafka, kafkaDsConfsWithEvents // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) // Assertions assert.NoError(t, err) @@ -121,7 +121,7 @@ func TestBuild_ProviderError(t *testing.T) { mockBuilder.On("BuildProvider", natsEventSources[0]).Return(nil, errors.New("provider error")) // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) // Assertions assert.Error(t, err) @@ -176,7 +176,7 @@ func TestBuild_ShouldGetAnErrorIfProviderIsNotDefined(t *testing.T) { mockBuilder.On("TypeID").Return("nats") // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) // Assertions assert.Error(t, err) @@ -239,7 +239,7 @@ func TestBuild_ShouldNotInitializeProviderIfNotUsed(t *testing.T) { mockBuilder.On("BuildProvider", natsEventSources[1]).Return(mockPubSubUsedProvider, nil) // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) // Assertions assert.NoError(t, err) @@ -290,7 +290,7 @@ func TestBuildProvidersAndDataSources_Nats_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr") + }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) // Assertions assert.NoError(t, err) @@ -343,7 +343,7 @@ func TestBuildProvidersAndDataSources_Kafka_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr") + }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) // Assertions assert.NoError(t, err) @@ -396,7 +396,7 @@ func TestBuildProvidersAndDataSources_Redis_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr") + }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) // Assertions assert.NoError(t, err) From 73c3fb2f15bc740d9a0e894f1d2f2fd757418403 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 17 Jul 2025 18:15:07 +0200 Subject: [PATCH 050/173] feat: add WriteEvent to context --- .../modules/start-subscription/module.go | 24 +++++++++++++++++++ .../modules/start_subscription_test.go | 23 +++++++++++++++--- router/core/streams_modules.go | 22 +++++++++++------ router/pkg/pubsub/datasource/datasource.go | 7 +++++- .../pkg/pubsub/datasource/hookeddatasource.go | 8 +++++-- router/pkg/pubsub/datasource/provider.go | 6 +++-- router/pkg/pubsub/kafka/engine_datasource.go | 23 ++++++++++++++---- .../pubsub/kafka/engine_datasource_factory.go | 2 +- router/pkg/pubsub/nats/engine_datasource.go | 23 ++++++++++++++---- .../pubsub/nats/engine_datasource_factory.go | 2 +- router/pkg/pubsub/redis/engine_datasource.go | 23 ++++++++++++++---- .../pubsub/redis/engine_datasource_factory.go | 2 +- 12 files changed, 132 insertions(+), 33 deletions(-) diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go index 2a062f02a3..81768d6a7a 100644 --- a/router-tests/modules/start-subscription/module.go +++ b/router-tests/modules/start-subscription/module.go @@ -1,9 +1,13 @@ package start_subscription import ( + "strings" + "go.uber.org/zap" "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) const myModuleID = "startSubscriptionModule" @@ -23,6 +27,26 @@ func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnSta m.Logger.Info("SubscriptionOnStart Hook has been run") + // check if the provider is nats + if ctx.SubscriptionEventConfiguration().ProviderType() != datasource.ProviderTypeKafka { + return nil + } + + // check if the provider id is the one expected by the module + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-kafka" { + return nil + } + + // check if the subject is the one expected by the module + kafkaConfig := ctx.SubscriptionEventConfiguration().(*kafka.SubscriptionEventConfiguration) + if !strings.Contains(kafkaConfig.Topics[0], "employeeUpdated") { + return nil + } + + ctx.WriteEvent(&kafka.Event{ + Data: []byte(`{"id": 1, "__typename": "Employee"}`), + }) + return nil } diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index b16f057a74..b86f04e7ec 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -47,14 +47,25 @@ func TestStartSubscriptionHook(t *testing.T) { Forename string `graphql:"forename"` Surname string `graphql:"surname"` } `graphql:"details"` - } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } `graphql:"employeeUpdatedMyKafka(employeeID: $employeeID)"` } surl := xEnv.GraphQLWebSocketSubscriptionURL() client := graphql.NewSubscriptionClient(surl) - subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { - // should never be called + vars := map[string]interface{}{ + "employeeID": 3, + } + type kafkaSubscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, vars, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } return nil }) require.NoError(t, err) @@ -67,9 +78,15 @@ func TestStartSubscriptionHook(t *testing.T) { xEnv.WaitForSubscriptionCount(1, time.Second*10) + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) + }) + require.NoError(t, client.Close()) testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { require.NoError(t, err) + }, "unable to close client before timeout") requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") diff --git a/router/core/streams_modules.go b/router/core/streams_modules.go index 264a66942b..f750c3a2d2 100644 --- a/router/core/streams_modules.go +++ b/router/core/streams_modules.go @@ -11,6 +11,10 @@ type StreamHookError struct { CloseSubscription bool } +func (e *StreamHookError) Error() string { + return e.HttpError.Error() +} + type SubscriptionOnStartHookContext interface { // the request context RequestContext() RequestContext @@ -23,6 +27,7 @@ type SubscriptionOnStartHookContext interface { type subscriptionOnStartHookContext struct { requestContext RequestContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration + events []datasource.StreamEvent } func (c *subscriptionOnStartHookContext) RequestContext() RequestContext { @@ -34,7 +39,7 @@ func (c *subscriptionOnStartHookContext) SubscriptionEventConfiguration() dataso } func (c *subscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - //c.subscriptionEventConfiguration.WriteEvent(event) + c.events = append(c.events, event) } type SubscriptionOnStartHandler interface { @@ -45,13 +50,16 @@ type SubscriptionOnStartHandler interface { //write a method that converts from func(ctx SubscriptionOnStartHookContext) error to func(ctx *resolve.Context, event StreamEvent) error -func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, event datasource.StreamEvent) error { - return func(resolveCtx *resolve.Context, event datasource.StreamEvent) error { +func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error, []datasource.StreamEvent) { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error, []datasource.StreamEvent) { requestContext := getRequestContext(resolveCtx.Context()) + hookCtx := &subscriptionOnStartHookContext{ + requestContext: requestContext, + subscriptionEventConfiguration: subConf, + } + + err := fn(hookCtx) - return fn(&subscriptionOnStartHookContext{ - requestContext: requestContext, - //subscriptionEventConfiguration: subscriptionEventConfiguration, - }) + return err, hookCtx.events } } diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index 2f08b97074..fbbc968350 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -4,6 +4,11 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) +type SubscriptionDataSourceWithConfiguration interface { + SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration + resolve.SubscriptionDataSource +} + // EngineDataSourceFactory is the interface that all pubsub data sources must implement. // It serves three main purposes: // 1. Resolving the data source and subscription data source @@ -23,7 +28,7 @@ type EngineDataSourceFactory interface { // ResolveDataSourceSubscription returns the engine SubscriptionDataSource implementation // that contains methods to start a subscription, which will be called by the Planner // when a subscription is initiated - ResolveDataSourceSubscription() (resolve.SubscriptionDataSource, error) + ResolveDataSourceSubscription() (SubscriptionDataSourceWithConfiguration, error) // ResolveDataSourceSubscriptionInput build the input that will be passed to the engine SubscriptionDataSource ResolveDataSourceSubscriptionInput() (string, error) // TransformEventData allows the data source to transform the event data using the extractFn diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 23ee76277b..1b9ed0e9f5 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -7,14 +7,18 @@ import ( type HookedSubscriptionDataSource struct { OnSubscriptionStartFns []OnSubscriptionStartFn - SubscriptionDataSource resolve.SubscriptionDataSource + SubscriptionDataSource SubscriptionDataSourceWithConfiguration } func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { for _, fn := range h.OnSubscriptionStartFns { - if err := fn(ctx, input); err != nil { + err, events := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) + if err != nil { return err } + for _, event := range events { + updater.Update(event.GetData()) + } } return h.SubscriptionDataSource.Start(ctx, input, updater) } diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 2aeaaa421d..1761c6abbb 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -46,9 +46,11 @@ const ( // StreamEvent is a generic interface for all stream events // Each provider will have its own event type that implements this interface // there could be common fields in future, but for now we don't need any -type StreamEvent interface{} +type StreamEvent interface { + GetData() []byte +} -type OnSubscriptionStartFn func(ctx *resolve.Context, event StreamEvent) error +type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (error, []StreamEvent) // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 2b29ba9899..f9d26309f7 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -21,6 +21,10 @@ type Event struct { Headers map[string][]byte `json:"headers"` } +func (e *Event) GetData() []byte { + return e.Data +} + type SubscriptionEventConfiguration struct { Provider string `json:"providerId"` Topics []string `json:"topics"` @@ -74,6 +78,15 @@ type SubscriptionDataSource struct { pubSub Adapter } +func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { + var subscriptionConfiguration SubscriptionEventConfiguration + err := json.Unmarshal(input, &subscriptionConfiguration) + if err != nil { + return nil + } + return &subscriptionConfiguration +} + func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { val, _, _, err := jsonparser.Get(input, "topics") if err != nil { @@ -95,13 +108,13 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b } func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return err + subConf := s.SubscriptionEventConfiguration(input) + conf, ok := subConf.(*SubscriptionEventConfiguration) + if !ok { + return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), subscriptionConfiguration, updater) + return s.pubSub.Subscribe(ctx.Context(), *conf, updater) } type PublishDataSource struct { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index 95589d6f9e..462207e516 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -58,7 +58,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return evtCfg.MarshalJSONTemplate(), nil } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.SubscriptionDataSource, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { return &SubscriptionDataSource{ pubSub: c.KafkaAdapter, }, nil diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 5e5227028f..1a99a291a9 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -20,6 +20,10 @@ type Event struct { Metadata map[string]string `json:"metadata"` } +func (e *Event) GetData() []byte { + return e.Data +} + type StreamConfiguration struct { Consumer string `json:"consumer"` ConsumerInactiveThreshold int32 `json:"consumerInactiveThreshold"` @@ -80,6 +84,15 @@ type SubscriptionSource struct { pubSub Adapter } +func (s *SubscriptionSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { + var subscriptionConfiguration SubscriptionEventConfiguration + err := json.Unmarshal(input, &subscriptionConfiguration) + if err != nil { + return nil + } + return &subscriptionConfiguration +} + func (s *SubscriptionSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { val, _, _, err := jsonparser.Get(input, "subjects") @@ -102,13 +115,13 @@ func (s *SubscriptionSource) UniqueRequestID(ctx *resolve.Context, input []byte, } func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return err + subConf := s.SubscriptionEventConfiguration(input) + conf, ok := subConf.(*SubscriptionEventConfiguration) + if !ok { + return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), subscriptionConfiguration, updater) + return s.pubSub.Subscribe(ctx.Context(), *conf, updater) } type NatsPublishDataSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index a8aa031fbc..a2cba7e04d 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -72,7 +72,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return evtCfg.MarshalJSONTemplate(), nil } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.SubscriptionDataSource, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { return &SubscriptionSource{ pubSub: c.NatsAdapter, }, nil diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 72ab322d88..f4497b6384 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -19,6 +19,10 @@ type Event struct { Data json.RawMessage `json:"data"` } +func (e *Event) GetData() []byte { + return e.Data +} + // SubscriptionEventConfiguration contains configuration for subscription events type SubscriptionEventConfiguration struct { Provider string `json:"providerId"` @@ -73,6 +77,15 @@ type SubscriptionDataSource struct { pubSub Adapter } +func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { + var subscriptionConfiguration SubscriptionEventConfiguration + err := json.Unmarshal(input, &subscriptionConfiguration) + if err != nil { + return nil + } + return &subscriptionConfiguration +} + // UniqueRequestID computes a unique ID for the subscription request func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { val, _, _, err := jsonparser.Get(input, "channels") @@ -96,13 +109,13 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b // Start starts the subscription func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return err + subConf := s.SubscriptionEventConfiguration(input) + conf, ok := subConf.(*SubscriptionEventConfiguration) + if !ok { + return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), subscriptionConfiguration, updater) + return s.pubSub.Subscribe(ctx.Context(), *conf, updater) } // LoadInitialData implements the interface method (not used for this subscription type) diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index 0c62bed033..1b0bc6652c 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -69,7 +69,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri } // ResolveDataSourceSubscription returns the subscription data source -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.SubscriptionDataSource, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { return &SubscriptionDataSource{ pubSub: c.RedisAdapter, }, nil From 8b3d17bc75695b2cba26949c7fb8db75f6ff8f98 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 18 Jul 2025 10:41:42 +0200 Subject: [PATCH 051/173] chore: add test, improve method signature --- .../modules/start-subscription/module.go | 27 +-- .../modules/start_subscription_test.go | 175 ++++++++++++++++++ router/core/streams_modules.go | 6 +- .../pkg/pubsub/datasource/hookeddatasource.go | 2 +- router/pkg/pubsub/datasource/provider.go | 2 +- 5 files changed, 184 insertions(+), 28 deletions(-) diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go index 81768d6a7a..6ece70d09c 100644 --- a/router-tests/modules/start-subscription/module.go +++ b/router-tests/modules/start-subscription/module.go @@ -1,19 +1,16 @@ package start_subscription import ( - "strings" - "go.uber.org/zap" "github.com/wundergraph/cosmo/router/core" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) const myModuleID = "startSubscriptionModule" type StartSubscriptionModule struct { - Logger *zap.Logger + Logger *zap.Logger + Callback func(ctx core.SubscriptionOnStartHookContext) error } func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { @@ -27,26 +24,10 @@ func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnSta m.Logger.Info("SubscriptionOnStart Hook has been run") - // check if the provider is nats - if ctx.SubscriptionEventConfiguration().ProviderType() != datasource.ProviderTypeKafka { - return nil - } - - // check if the provider id is the one expected by the module - if ctx.SubscriptionEventConfiguration().ProviderID() != "my-kafka" { - return nil + if m.Callback != nil { + return m.Callback(ctx) } - // check if the subject is the one expected by the module - kafkaConfig := ctx.SubscriptionEventConfiguration().(*kafka.SubscriptionEventConfiguration) - if !strings.Contains(kafkaConfig.Topics[0], "employeeUpdated") { - return nil - } - - ctx.WriteEvent(&kafka.Event{ - Data: []byte(`{"id": 1, "__typename": "Employee"}`), - }) - return nil } diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index b86f04e7ec..bdccb4ccc6 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -1,6 +1,7 @@ package module_test import ( + "errors" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/core" "github.com/wundergraph/cosmo/router/pkg/config" + "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) func TestStartSubscriptionHook(t *testing.T) { @@ -28,6 +30,75 @@ func TestStartSubscriptionHook(t *testing.T) { }, } + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: $employeeID)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "employeeID": 3, + } + subscriptionOneID, err := client.Subscribe(&subscriptionOne, vars, func(dataValue []byte, errValue error) error { + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) + + t.Run("Test StartSubscription write event works", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) error { + ctx.WriteEvent(&kafka.Event{ + Data: []byte(`{"id": 1, "__typename": "Employee"}`), + }) + return nil + }, + }, + }, + } + testenv.Run(t, &testenv.Config{ RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, @@ -93,4 +164,108 @@ func TestStartSubscriptionHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) + + t.Run("Test StartSubscription write event sends event only to the subscription", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) error { + employeeId := ctx.RequestContext().Operation().Variables().GetInt64("employeeID") + if employeeId != 1 { + return nil + } + ctx.WriteEvent(&kafka.Event{ + Data: []byte(`{"id": 1, "__typename": "Employee"}`), + }) + return nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscription struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: $employeeID)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "employeeID": 3, + } + vars2 := map[string]interface{}{ + "employeeID": 1, + } + type kafkaSubscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionOneArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscription, vars, func(dataValue []byte, errValue error) error { + subscriptionOneArgsCh <- kafkaSubscriptionArgs{ + dataValue: []byte{}, + errValue: errors.New("should not be called"), + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + subscriptionTwoArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionTwoID, err := client.Subscribe(&subscription, vars2, func(dataValue []byte, errValue error) error { + subscriptionTwoArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionTwoID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(2, time.Second*10) + + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionTwoArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + t.Cleanup(func() { + require.Len(t, subscriptionOneArgsCh, 0) + }) + }) + }) } diff --git a/router/core/streams_modules.go b/router/core/streams_modules.go index f750c3a2d2..51953e318a 100644 --- a/router/core/streams_modules.go +++ b/router/core/streams_modules.go @@ -50,8 +50,8 @@ type SubscriptionOnStartHandler interface { //write a method that converts from func(ctx SubscriptionOnStartHookContext) error to func(ctx *resolve.Context, event StreamEvent) error -func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error, []datasource.StreamEvent) { - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error, []datasource.StreamEvent) { +func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &subscriptionOnStartHookContext{ requestContext: requestContext, @@ -60,6 +60,6 @@ func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) err := fn(hookCtx) - return err, hookCtx.events + return hookCtx.events, err } } diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 1b9ed0e9f5..de7a5b204a 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -12,7 +12,7 @@ type HookedSubscriptionDataSource struct { func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { for _, fn := range h.OnSubscriptionStartFns { - err, events := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) + events, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil { return err } diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 1761c6abbb..a19c0eb937 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -50,7 +50,7 @@ type StreamEvent interface { GetData() []byte } -type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (error, []StreamEvent) +type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) ([]StreamEvent, error) // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { From 17984ce14f625e4b1cc4adc271531bca6663af4f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 18 Jul 2025 16:51:37 +0200 Subject: [PATCH 052/173] feat: pass down additional event data to pubsub system --- .../modules/start_subscription_test.go | 1 + router/pkg/pubsub/datasource/datasource.go | 8 +- .../pkg/pubsub/datasource/hookeddatasource.go | 7 +- router/pkg/pubsub/datasource/mocks.go | 156 +++++++++++++++++- .../datasource/subscription_event_updater.go | 31 ++++ router/pkg/pubsub/kafka/adapter.go | 18 +- router/pkg/pubsub/kafka/engine_datasource.go | 16 +- .../pubsub/kafka/engine_datasource_factory.go | 4 +- .../pubsub/kafka/engine_datasource_test.go | 35 +++- router/pkg/pubsub/kafka/mocks.go | 16 +- router/pkg/pubsub/nats/adapter.go | 13 +- router/pkg/pubsub/nats/engine_datasource.go | 2 +- .../pubsub/nats/engine_datasource_factory.go | 2 +- .../pkg/pubsub/nats/engine_datasource_test.go | 10 +- router/pkg/pubsub/nats/mocks.go | 16 +- router/pkg/pubsub/redis/adapter.go | 9 +- router/pkg/pubsub/redis/engine_datasource.go | 2 +- .../pubsub/redis/engine_datasource_factory.go | 2 +- .../pubsub/redis/engine_datasource_test.go | 10 +- router/pkg/pubsub/redis/mocks.go | 16 +- 20 files changed, 295 insertions(+), 79 deletions(-) create mode 100644 router/pkg/pubsub/datasource/subscription_event_updater.go diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index bdccb4ccc6..747f1ebeab 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -91,6 +91,7 @@ func TestStartSubscriptionHook(t *testing.T) { "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { ctx.WriteEvent(&kafka.Event{ + Key: []byte("1"), Data: []byte(`{"id": 1, "__typename": "Employee"}`), }) return nil diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index fbbc968350..c61909f495 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -1,12 +1,14 @@ package datasource import ( + "github.com/cespare/xxhash/v2" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -type SubscriptionDataSourceWithConfiguration interface { +type PubSubSubscriptionDataSource interface { SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration - resolve.SubscriptionDataSource + Start(ctx *resolve.Context, input []byte, updater SubscriptionEventUpdater) error + UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) } // EngineDataSourceFactory is the interface that all pubsub data sources must implement. @@ -28,7 +30,7 @@ type EngineDataSourceFactory interface { // ResolveDataSourceSubscription returns the engine SubscriptionDataSource implementation // that contains methods to start a subscription, which will be called by the Planner // when a subscription is initiated - ResolveDataSourceSubscription() (SubscriptionDataSourceWithConfiguration, error) + ResolveDataSourceSubscription() (PubSubSubscriptionDataSource, error) // ResolveDataSourceSubscriptionInput build the input that will be passed to the engine SubscriptionDataSource ResolveDataSourceSubscriptionInput() (string, error) // TransformEventData allows the data source to transform the event data using the extractFn diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index de7a5b204a..6f24e1aab8 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -7,20 +7,21 @@ import ( type HookedSubscriptionDataSource struct { OnSubscriptionStartFns []OnSubscriptionStartFn - SubscriptionDataSource SubscriptionDataSourceWithConfiguration + SubscriptionDataSource PubSubSubscriptionDataSource } func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { + subscriptionEventUpdater := NewSubscriptionEventUpdater(updater) for _, fn := range h.OnSubscriptionStartFns { events, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil { return err } for _, event := range events { - updater.Update(event.GetData()) + subscriptionEventUpdater.Update(event) } } - return h.SubscriptionDataSource.Start(ctx, input, updater) + return h.SubscriptionDataSource.Start(ctx, input, subscriptionEventUpdater) } func (h *HookedSubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) { diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index 067da9c86c..1968b4bba1 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -198,23 +198,23 @@ func (_c *MockEngineDataSourceFactory_ResolveDataSourceInput_Call) RunAndReturn( } // ResolveDataSourceSubscription provides a mock function for the type MockEngineDataSourceFactory -func (_mock *MockEngineDataSourceFactory) ResolveDataSourceSubscription() (resolve.SubscriptionDataSource, error) { +func (_mock *MockEngineDataSourceFactory) ResolveDataSourceSubscription() (PubSubSubscriptionDataSource, error) { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for ResolveDataSourceSubscription") } - var r0 resolve.SubscriptionDataSource + var r0 PubSubSubscriptionDataSource var r1 error - if returnFunc, ok := ret.Get(0).(func() (resolve.SubscriptionDataSource, error)); ok { + if returnFunc, ok := ret.Get(0).(func() (PubSubSubscriptionDataSource, error)); ok { return returnFunc() } - if returnFunc, ok := ret.Get(0).(func() resolve.SubscriptionDataSource); ok { + if returnFunc, ok := ret.Get(0).(func() PubSubSubscriptionDataSource); ok { r0 = returnFunc() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(resolve.SubscriptionDataSource) + r0 = ret.Get(0).(PubSubSubscriptionDataSource) } } if returnFunc, ok := ret.Get(1).(func() error); ok { @@ -242,12 +242,12 @@ func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Run(ru return _c } -func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Return(subscriptionDataSource resolve.SubscriptionDataSource, err error) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { - _c.Call.Return(subscriptionDataSource, err) +func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Return(pubSubSubscriptionDataSource PubSubSubscriptionDataSource, err error) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { + _c.Call.Return(pubSubSubscriptionDataSource, err) return _c } -func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) RunAndReturn(run func() (resolve.SubscriptionDataSource, error)) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { +func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) RunAndReturn(run func() (PubSubSubscriptionDataSource, error)) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { _c.Call.Return(run) return _c } @@ -896,3 +896,143 @@ func (_c *MockProviderBuilder_TypeID_Call[P, E]) RunAndReturn(run func() string) _c.Call.Return(run) return _c } + +// NewMockSubscriptionEventUpdater creates a new instance of MockSubscriptionEventUpdater. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockSubscriptionEventUpdater(t interface { + mock.TestingT + Cleanup(func()) +}) *MockSubscriptionEventUpdater { + mock := &MockSubscriptionEventUpdater{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockSubscriptionEventUpdater is an autogenerated mock type for the SubscriptionEventUpdater type +type MockSubscriptionEventUpdater struct { + mock.Mock +} + +type MockSubscriptionEventUpdater_Expecter struct { + mock *mock.Mock +} + +func (_m *MockSubscriptionEventUpdater) EXPECT() *MockSubscriptionEventUpdater_Expecter { + return &MockSubscriptionEventUpdater_Expecter{mock: &_m.Mock} +} + +// Close provides a mock function for the type MockSubscriptionEventUpdater +func (_mock *MockSubscriptionEventUpdater) Close(kind resolve.SubscriptionCloseKind) { + _mock.Called(kind) + return +} + +// MockSubscriptionEventUpdater_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type MockSubscriptionEventUpdater_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +// - kind resolve.SubscriptionCloseKind +func (_e *MockSubscriptionEventUpdater_Expecter) Close(kind interface{}) *MockSubscriptionEventUpdater_Close_Call { + return &MockSubscriptionEventUpdater_Close_Call{Call: _e.mock.On("Close", kind)} +} + +func (_c *MockSubscriptionEventUpdater_Close_Call) Run(run func(kind resolve.SubscriptionCloseKind)) *MockSubscriptionEventUpdater_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 resolve.SubscriptionCloseKind + if args[0] != nil { + arg0 = args[0].(resolve.SubscriptionCloseKind) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSubscriptionEventUpdater_Close_Call) Return() *MockSubscriptionEventUpdater_Close_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionEventUpdater_Close_Call) RunAndReturn(run func(kind resolve.SubscriptionCloseKind)) *MockSubscriptionEventUpdater_Close_Call { + _c.Run(run) + return _c +} + +// Complete provides a mock function for the type MockSubscriptionEventUpdater +func (_mock *MockSubscriptionEventUpdater) Complete() { + _mock.Called() + return +} + +// MockSubscriptionEventUpdater_Complete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Complete' +type MockSubscriptionEventUpdater_Complete_Call struct { + *mock.Call +} + +// Complete is a helper method to define mock.On call +func (_e *MockSubscriptionEventUpdater_Expecter) Complete() *MockSubscriptionEventUpdater_Complete_Call { + return &MockSubscriptionEventUpdater_Complete_Call{Call: _e.mock.On("Complete")} +} + +func (_c *MockSubscriptionEventUpdater_Complete_Call) Run(run func()) *MockSubscriptionEventUpdater_Complete_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSubscriptionEventUpdater_Complete_Call) Return() *MockSubscriptionEventUpdater_Complete_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionEventUpdater_Complete_Call) RunAndReturn(run func()) *MockSubscriptionEventUpdater_Complete_Call { + _c.Run(run) + return _c +} + +// Update provides a mock function for the type MockSubscriptionEventUpdater +func (_mock *MockSubscriptionEventUpdater) Update(event StreamEvent) { + _mock.Called(event) + return +} + +// MockSubscriptionEventUpdater_Update_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Update' +type MockSubscriptionEventUpdater_Update_Call struct { + *mock.Call +} + +// Update is a helper method to define mock.On call +// - event StreamEvent +func (_e *MockSubscriptionEventUpdater_Expecter) Update(event interface{}) *MockSubscriptionEventUpdater_Update_Call { + return &MockSubscriptionEventUpdater_Update_Call{Call: _e.mock.On("Update", event)} +} + +func (_c *MockSubscriptionEventUpdater_Update_Call) Run(run func(event StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 StreamEvent + if args[0] != nil { + arg0 = args[0].(StreamEvent) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSubscriptionEventUpdater_Update_Call) Return() *MockSubscriptionEventUpdater_Update_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(event StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { + _c.Run(run) + return _c +} diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go new file mode 100644 index 0000000000..33e71da8ee --- /dev/null +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -0,0 +1,31 @@ +package datasource + +import "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + +type SubscriptionEventUpdater interface { + Update(event StreamEvent) + Complete() + Close(kind resolve.SubscriptionCloseKind) +} + +type subscriptionEventUpdater struct { + eventUpdater resolve.SubscriptionUpdater +} + +func (h *subscriptionEventUpdater) Update(event StreamEvent) { + h.eventUpdater.Update(event.GetData()) +} + +func (h *subscriptionEventUpdater) Complete() { + h.eventUpdater.Complete() +} + +func (h *subscriptionEventUpdater) Close(kind resolve.SubscriptionCloseKind) { + h.eventUpdater.Close(kind) +} + +func NewSubscriptionEventUpdater(eventUpdater resolve.SubscriptionUpdater) SubscriptionEventUpdater { + return &subscriptionEventUpdater{ + eventUpdater: eventUpdater, + } +} diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 9ba0308582..a100a3ebdf 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -11,7 +11,6 @@ import ( "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kgo" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) @@ -21,7 +20,7 @@ var ( // Adapter defines the interface for Kafka adapter operations type Adapter interface { - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error + Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error Publish(ctx context.Context, event PublishEventConfiguration) error Startup(ctx context.Context) error Shutdown(ctx context.Context) error @@ -42,7 +41,7 @@ type ProviderAdapter struct { } // topicPoller polls the Kafka topic for new records and calls the updateTriggers function. -func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, updater resolve.SubscriptionUpdater) error { +func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, updater datasource.SubscriptionEventUpdater) error { for { select { case <-p.ctx.Done(): // Close the poller if the application context was canceled @@ -88,7 +87,16 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u r := iter.Next() p.logger.Debug("subscription update", zap.String("topic", r.Topic), zap.ByteString("data", r.Value)) - updater.Update(r.Value) + headers := make(map[string][]byte) + for _, header := range r.Headers { + headers[header.Key] = header.Value + } + + updater.Update(&Event{ + Data: r.Value, + Headers: headers, + Key: r.Key, + }) } } } @@ -96,7 +104,7 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u // Subscribe subscribes to the given topics and updates the subscription updater. // The engine already deduplicates subscriptions with the same topics, stream configuration, extensions, headers, etc. -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { log := p.logger.With( zap.String("provider_id", event.ProviderID()), diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index f9d26309f7..b7a9da1d45 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -68,10 +68,20 @@ func (p *PublishEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishEventConfiguration) MarshalJSONTemplate() string { +func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.ProviderID()) + headers := s.Event.Headers + if headers == nil { + headers = make(map[string][]byte) + } + + headersBytes, err := json.Marshal(headers) + if err != nil { + return "", err + } + + return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil } type SubscriptionDataSource struct { @@ -107,7 +117,7 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b return err } -func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { +func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index 462207e516..b4672ebfb4 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -55,10 +55,10 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri FieldName: c.fieldName, } - return evtCfg.MarshalJSONTemplate(), nil + return evtCfg.MarshalJSONTemplate() } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { return &SubscriptionDataSource{ pubSub: c.KafkaAdapter, }, nil diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 650153f3d1..881b7779e0 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -28,7 +28,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Topic: "test-topic", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, }, - wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, + wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}, "key": "", "headers": {}}, "providerId":"test-provider"}`, }, { name: "with special characters", @@ -37,13 +37,32 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Topic: "topic-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, }, - wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}, "key": "", "headers": {}}, "providerId":"test-provider-id"}`, + }, + { + name: "with key", + config: PublishEventConfiguration{ + Provider: "test-provider-id", + Topic: "topic-with-hyphens", + Event: Event{Key: []byte("blablabla"), Data: json.RawMessage(`{}`)}, + }, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "blablabla", "headers": {}}, "providerId":"test-provider-id"}`, + }, + { + name: "with headers", + config: PublishEventConfiguration{ + Provider: "test-provider-id", + Topic: "topic-with-hyphens", + Event: Event{Headers: map[string][]byte{"key": []byte(`blablabla`)}, Data: json.RawMessage(`{}`)}, + }, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "", "headers": {"key":"YmxhYmxhYmxh"}}, "providerId":"test-provider-id"}`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := tt.config.MarshalJSONTemplate() + result, err := tt.config.MarshalJSONTemplate() + assert.NoError(t, err) assert.Equal(t, tt.wantPattern, result) }) } @@ -105,13 +124,13 @@ func TestSubscriptionSource_Start(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionUpdater) + mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) expectError bool }{ { name: "successful subscription", input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1", "topic2"}, @@ -122,7 +141,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "adapter returns error", input: `{"topics":["topic1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1"}, @@ -133,7 +152,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) {}, + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, expectError: true, }, } @@ -141,7 +160,7 @@ func TestSubscriptionSource_Start(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionUpdater(t) + updater := datasource.NewMockSubscriptionEventUpdater(t) tt.mockSetup(mockAdapter, updater) source := &SubscriptionDataSource{ diff --git a/router/pkg/pubsub/kafka/mocks.go b/router/pkg/pubsub/kafka/mocks.go index f39aee8b4e..da945393bc 100644 --- a/router/pkg/pubsub/kafka/mocks.go +++ b/router/pkg/pubsub/kafka/mocks.go @@ -8,7 +8,7 @@ import ( "context" mock "github.com/stretchr/testify/mock" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" ) // NewMockAdapter creates a new instance of MockAdapter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -198,7 +198,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -206,7 +206,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, resolve.SubscriptionUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -222,12 +222,12 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context // - event SubscriptionEventConfiguration -// - updater resolve.SubscriptionUpdater +// - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -237,9 +237,9 @@ func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event Su if args[1] != nil { arg1 = args[1].(SubscriptionEventConfiguration) } - var arg2 resolve.SubscriptionUpdater + var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { - arg2 = args[2].(resolve.SubscriptionUpdater) + arg2 = args[2].(datasource.SubscriptionEventUpdater) } run( arg0, @@ -255,7 +255,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 2b9a890498..bde0ec1bbe 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -12,14 +12,13 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) // Adapter defines the methods that a NATS adapter should implement type Adapter interface { // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error + Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error // Publish publishes the given event to the specified subject Publish(ctx context.Context, event PublishAndRequestEventConfiguration) error // Request sends a request to the specified subject and writes the response to the given writer @@ -72,7 +71,7 @@ func (p *ProviderAdapter) getDurableConsumerName(durableName string, subjects [] return fmt.Sprintf("%s-%x", durableName, subjHash.Sum64()), nil } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { log := p.logger.With( zap.String("provider_id", event.ProviderID()), zap.String("method", "subscribe"), @@ -132,7 +131,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) - updater.Update(msg.Data()) + updater.Update(&Event{ + Data: msg.Data(), + }) // Acknowledge the message after it has been processed ackErr := msg.Ack() @@ -169,7 +170,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent select { case msg := <-msgChan: log.Debug("subscription update", zap.String("message_subject", msg.Subject), zap.ByteString("data", msg.Data)) - updater.Update(msg.Data) + updater.Update(&Event{ + Data: msg.Data, + }) case <-p.ctx.Done(): // When the application context is done, we stop the subscriptions for _, subscription := range subscriptions { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 1a99a291a9..c3f910dd0e 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -114,7 +114,7 @@ func (s *SubscriptionSource) UniqueRequestID(ctx *resolve.Context, input []byte, return err } -func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { +func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index a2cba7e04d..f37c1dc8cc 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -72,7 +72,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return evtCfg.MarshalJSONTemplate(), nil } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { return &SubscriptionSource{ pubSub: c.NatsAdapter, }, nil diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index b243e27536..5213b21263 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -106,13 +106,13 @@ func TestSubscriptionSource_Start(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionUpdater) + mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) expectError bool }{ { name: "successful subscription", input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Subjects: []string{"subject1", "subject2"}, @@ -123,7 +123,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "adapter returns error", input: `{"subjects":["subject1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Subjects: []string{"subject1"}, @@ -134,7 +134,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) {}, + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, expectError: true, }, } @@ -142,7 +142,7 @@ func TestSubscriptionSource_Start(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionUpdater(t) + updater := datasource.NewMockSubscriptionEventUpdater(t) tt.mockSetup(mockAdapter, updater) source := &SubscriptionSource{ diff --git a/router/pkg/pubsub/nats/mocks.go b/router/pkg/pubsub/nats/mocks.go index de49c6ae7e..8c356b7b1c 100644 --- a/router/pkg/pubsub/nats/mocks.go +++ b/router/pkg/pubsub/nats/mocks.go @@ -9,7 +9,7 @@ import ( "io" mock "github.com/stretchr/testify/mock" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" ) // NewMockAdapter creates a new instance of MockAdapter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -262,7 +262,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -270,7 +270,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, resolve.SubscriptionUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -286,12 +286,12 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context // - event SubscriptionEventConfiguration -// - updater resolve.SubscriptionUpdater +// - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -301,9 +301,9 @@ func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event Su if args[1] != nil { arg1 = args[1].(SubscriptionEventConfiguration) } - var arg2 resolve.SubscriptionUpdater + var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { - arg2 = args[2].(resolve.SubscriptionUpdater) + arg2 = args[2].(datasource.SubscriptionEventUpdater) } run( arg0, @@ -319,7 +319,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 9c99f4f173..556e676048 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -7,14 +7,13 @@ import ( rd "github.com/wundergraph/cosmo/router/internal/persistedoperation/operationstorage/redis" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) // Adapter defines the methods that a Redis adapter should implement type Adapter interface { // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error + Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error // Publish publishes the given event to the specified channel Publish(ctx context.Context, event PublishEventConfiguration) error // Startup initializes the adapter @@ -74,7 +73,7 @@ func (p *ProviderAdapter) Shutdown(ctx context.Context) error { return p.conn.Close() } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { log := p.logger.With( zap.String("provider_id", event.ProviderID()), zap.String("method", "subscribe"), @@ -107,7 +106,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return } log.Debug("subscription update", zap.String("message_channel", msg.Channel), zap.String("data", msg.Payload)) - updater.Update([]byte(msg.Payload)) + updater.Update(&Event{ + Data: []byte(msg.Payload), + }) case <-p.ctx.Done(): // When the application context is done, we stop the subscription if it is not already done log.Debug("application context done, stopping subscription") diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index f4497b6384..51ad0963c3 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -108,7 +108,7 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b } // Start starts the subscription -func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { +func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index 1b0bc6652c..0db9b5a8f4 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -69,7 +69,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri } // ResolveDataSourceSubscription returns the subscription data source -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSourceWithConfiguration, error) { +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { return &SubscriptionDataSource{ pubSub: c.RedisAdapter, }, nil diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index e20d326aba..a343e51503 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -106,13 +106,13 @@ func TestSubscriptionSource_Start(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionUpdater) + mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) expectError bool }{ { name: "successful subscription", input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1", "channel2"}, @@ -123,7 +123,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "adapter returns error", input: `{"channels":["channel1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) { + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1"}, @@ -134,7 +134,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionUpdater) {}, + mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, expectError: true, }, } @@ -142,7 +142,7 @@ func TestSubscriptionSource_Start(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionUpdater(t) + updater := datasource.NewMockSubscriptionEventUpdater(t) tt.mockSetup(mockAdapter, updater) source := &SubscriptionDataSource{ diff --git a/router/pkg/pubsub/redis/mocks.go b/router/pkg/pubsub/redis/mocks.go index 603a5dd548..91c6ca9205 100644 --- a/router/pkg/pubsub/redis/mocks.go +++ b/router/pkg/pubsub/redis/mocks.go @@ -8,7 +8,7 @@ import ( "context" mock "github.com/stretchr/testify/mock" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" ) // NewMockAdapter creates a new instance of MockAdapter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. @@ -198,7 +198,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -206,7 +206,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, resolve.SubscriptionUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -222,12 +222,12 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context // - event SubscriptionEventConfiguration -// - updater resolve.SubscriptionUpdater +// - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -237,9 +237,9 @@ func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event Su if args[1] != nil { arg1 = args[1].(SubscriptionEventConfiguration) } - var arg2 resolve.SubscriptionUpdater + var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { - arg2 = args[2].(resolve.SubscriptionUpdater) + arg2 = args[2].(datasource.SubscriptionEventUpdater) } run( arg0, @@ -255,7 +255,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater resolve.SubscriptionUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } From 0992ef632a4c2175199d38b92cedbe6b79b0972b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 18 Jul 2025 16:52:35 +0200 Subject: [PATCH 053/173] chore: add SubscriptionEventUpdater to mockery --- router/.mockery.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/router/.mockery.yml b/router/.mockery.yml index c835d3af50..8ea750cc0e 100644 --- a/router/.mockery.yml +++ b/router/.mockery.yml @@ -17,6 +17,7 @@ packages: ProviderBuilder: EngineDataSourceFactory: Provider: + SubscriptionEventUpdater: github.com/wundergraph/cosmo/router/pkg/pubsub/nats: interfaces: Adapter: From 66224598e91fc89a7feb93bbf8c65f2a63472553 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 18 Jul 2025 20:17:13 +0200 Subject: [PATCH 054/173] chore: improved structure, add support for start hook also on subgraph subscriptions --- .../modules/start_subscription_test.go | 139 ++++++++++++++++++ router/core/executor.go | 4 +- router/core/factoryresolver.go | 30 ++-- router/core/graph_server.go | 2 +- router/core/plan_generator.go | 2 +- router/core/router.go | 2 +- router/core/router_config.go | 6 +- router/core/streams_modules.go | 65 -------- router/core/subscriptions_modules.go | 104 +++++++++++++ 9 files changed, 270 insertions(+), 84 deletions(-) delete mode 100644 router/core/streams_modules.go create mode 100644 router/core/subscriptions_modules.go diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 747f1ebeab..1557dfad9a 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -269,4 +269,143 @@ func TestStartSubscriptionHook(t *testing.T) { }) }) }) + + t.Run("Test StartSubscription hook is called for engine subscription", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{}, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + + var subscriptionCountEmp struct { + CountEmp int `graphql:"countEmp(max: $max, intervalMilliseconds: $interval)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "max": 1, + "interval": 200, + } + subscriptionOneID, err := client.Subscribe(&subscriptionCountEmp, vars, func(dataValue []byte, errValue error) error { + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) + + t.Run("Test StartSubscription hook is called for engine subscription and write event works", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) error { + ctx.WriteEvent(&core.EngineEvent{ + Data: []byte(`{"data":{"countEmp":1000}}`), + }) + return nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + + var subscriptionCountEmp struct { + CountEmp int `graphql:"countEmp(max: $max, intervalMilliseconds: $interval)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "max": 0, + "interval": 0, + } + + type subscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionOneArgsCh := make(chan subscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionCountEmp, vars, func(dataValue []byte, errValue error) error { + subscriptionOneArgsCh <- subscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionOneArgsCh, func(t *testing.T, args subscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"countEmp": 1000}`, string(args.dataValue)) + }) + + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionOneArgsCh, func(t *testing.T, args subscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"countEmp": 0}`, string(args.dataValue)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) } diff --git a/router/core/executor.go b/router/core/executor.go index d8ee7145d0..80e241c62a 100644 --- a/router/core/executor.go +++ b/router/core/executor.go @@ -36,7 +36,7 @@ type ExecutorConfigurationBuilder struct { subscriptionClientOptions *SubscriptionClientOptions instanceData InstanceData - startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error + subscriptionHooks subscriptionHooks } type Executor struct { @@ -216,7 +216,7 @@ func (b *ExecutorConfigurationBuilder) buildPlannerConfiguration(ctx context.Con routerEngineCfg.Execution.EnableSingleFlight, routerEngineCfg.Execution.EnableNetPoll, b.instanceData, - ), b.logger, b.startSubscriptionModules) + ), b.logger, b.subscriptionHooks) // this generates the plan config using the data source factories from the config package planConfig, providers, err := loader.Load(engineConfig, subgraphs, routerEngineCfg, pluginsEnabled) diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 3eb75107ea..d07733ae39 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -29,9 +29,9 @@ import ( ) type Loader struct { - ctx context.Context - resolver FactoryResolver - startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error + ctx context.Context + resolver FactoryResolver + subscriptionHooks subscriptionHooks // includeInfo controls whether additional information like type usage and field usage is included in the plan de includeInfo bool logger *zap.Logger @@ -189,13 +189,13 @@ func (d *DefaultFactoryResolver) InstanceData() InstanceData { return d.instanceData } -func NewLoader(ctx context.Context, includeInfo bool, resolver FactoryResolver, logger *zap.Logger, startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error) *Loader { +func NewLoader(ctx context.Context, includeInfo bool, resolver FactoryResolver, logger *zap.Logger, subscriptionHooks subscriptionHooks) *Loader { return &Loader{ - ctx: ctx, - resolver: resolver, - includeInfo: includeInfo, - logger: logger, - startSubscriptionModules: startSubscriptionModules, + ctx: ctx, + resolver: resolver, + includeInfo: includeInfo, + logger: logger, + subscriptionHooks: subscriptionHooks, } } @@ -415,6 +415,10 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } + onSubscriptionStarts := make([]graphql_datasource.OnSubscriptionStartFn, len(l.subscriptionHooks.startSubscription)) + for i, fn := range l.subscriptionHooks.startSubscription { + onSubscriptionStarts[i] = NewEngineOnSubscriptionStartHook(fn) + } customConfiguration, err := graphql_datasource.NewConfiguration(graphql_datasource.ConfigurationInput{ Fetch: &graphql_datasource.FetchConfiguration{ URL: fetchUrl, @@ -428,6 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, + OnSubscriptionStartFns: onSubscriptionStarts, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, @@ -469,11 +474,10 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } - onSubscriptionStarts := make([]pubsub_datasource.OnSubscriptionStartFn, len(l.startSubscriptionModules)) - for i, fn := range l.startSubscriptionModules { - onSubscriptionStarts[i] = callSubscriptionOnStart(fn) + onSubscriptionStarts := make([]pubsub_datasource.OnSubscriptionStartFn, len(l.subscriptionHooks.startSubscription)) + for i, fn := range l.subscriptionHooks.startSubscription { + onSubscriptionStarts[i] = NewPubSubOnSubscriptionStartHook(fn) } - factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, routerEngineConfig.Events, diff --git a/router/core/graph_server.go b/router/core/graph_server.go index 839e6973a3..fed499ac5c 100644 --- a/router/core/graph_server.go +++ b/router/core/graph_server.go @@ -1164,7 +1164,7 @@ func (s *graphServer) buildGraphMux( EnableTraceClient: enableTraceClient, CircuitBreaker: s.circuitBreakerManager, }, - startSubscriptionModules: s.startSubscriptionModules, + subscriptionHooks: s.subscriptionHooks, } executor, providers, err := ecb.Build( diff --git a/router/core/plan_generator.go b/router/core/plan_generator.go index ba9c8b8072..a5bc8a6aba 100644 --- a/router/core/plan_generator.go +++ b/router/core/plan_generator.go @@ -282,7 +282,7 @@ func (pg *PlanGenerator) loadConfiguration(routerConfig *nodev1.RouterConfig, lo httpClient: http.DefaultClient, streamingClient: http.DefaultClient, subscriptionClient: subscriptionClient, - }, logger, nil) + }, logger, subscriptionHooks{}) // this generates the plan configuration using the data source factories from the config package planConfig, _, err := loader.Load(routerConfig.GetEngineConfig(), routerConfig.GetSubgraphs(), &routerEngineConfig, false) // TODO: configure plugins diff --git a/router/core/router.go b/router/core/router.go index bd023849a8..2f83d284e1 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -643,7 +643,7 @@ func (r *Router) initModules(ctx context.Context) error { } if handler, ok := moduleInstance.(SubscriptionOnStartHandler); ok { - r.startSubscriptionModules = append(r.startSubscriptionModules, handler.SubscriptionOnStart) + r.subscriptionHooks.startSubscription = append(r.subscriptionHooks.startSubscription, handler.SubscriptionOnStart) } r.modules = append(r.modules, moduleInstance) diff --git a/router/core/router_config.go b/router/core/router_config.go index 28c4a4481e..b97ba5eecb 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -25,6 +25,10 @@ import ( "go.uber.org/zap" ) +type subscriptionHooks struct { + startSubscription []func(ctx SubscriptionOnStartHookContext) error +} + type Config struct { clusterName string instanceID string @@ -118,7 +122,7 @@ type Config struct { mcp config.MCPConfiguration plugins config.PluginsConfiguration tracingAttributes []config.CustomAttribute - startSubscriptionModules []func(ctx SubscriptionOnStartHookContext) error + subscriptionHooks subscriptionHooks } // Usage returns an anonymized version of the config for usage tracking diff --git a/router/core/streams_modules.go b/router/core/streams_modules.go deleted file mode 100644 index 51953e318a..0000000000 --- a/router/core/streams_modules.go +++ /dev/null @@ -1,65 +0,0 @@ -package core - -import ( - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" -) - -// StreamHookError is used to customize the error messages and the behavior -type StreamHookError struct { - HttpError HttpError - CloseSubscription bool -} - -func (e *StreamHookError) Error() string { - return e.HttpError.Error() -} - -type SubscriptionOnStartHookContext interface { - // the request context - RequestContext() RequestContext - // the subscription event configuration - SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration - // write an event to the stream of the current subscription - WriteEvent(event datasource.StreamEvent) -} - -type subscriptionOnStartHookContext struct { - requestContext RequestContext - subscriptionEventConfiguration datasource.SubscriptionEventConfiguration - events []datasource.StreamEvent -} - -func (c *subscriptionOnStartHookContext) RequestContext() RequestContext { - return c.requestContext -} - -func (c *subscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { - return c.subscriptionEventConfiguration -} - -func (c *subscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - c.events = append(c.events, event) -} - -type SubscriptionOnStartHandler interface { - // OnSubscriptionOnStart is called once at subscription start - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error -} - -//write a method that converts from func(ctx SubscriptionOnStartHookContext) error to func(ctx *resolve.Context, event StreamEvent) error - -func callSubscriptionOnStart(fn func(ctx SubscriptionOnStartHookContext) error) func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { - requestContext := getRequestContext(resolveCtx.Context()) - hookCtx := &subscriptionOnStartHookContext{ - requestContext: requestContext, - subscriptionEventConfiguration: subConf, - } - - err := fn(hookCtx) - - return hookCtx.events, err - } -} diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go new file mode 100644 index 0000000000..86b74d2d81 --- /dev/null +++ b/router/core/subscriptions_modules.go @@ -0,0 +1,104 @@ +package core + +import ( + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +// SubscriptionHookError is used to customize the error messages and the behavior +type SubscriptionHookError struct { + HttpError HttpError + CloseSubscription bool +} + +func (e *SubscriptionHookError) Error() string { + return e.HttpError.Error() +} + +type SubscriptionOnStartHookContext interface { + // the request context + RequestContext() RequestContext + // the subscription event configuration + SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration + // write an event to the stream of the current subscription + WriteEvent(event datasource.StreamEvent) +} + +type pubSubSubscriptionOnStartHookContext struct { + requestContext RequestContext + subscriptionEventConfiguration datasource.SubscriptionEventConfiguration + events []datasource.StreamEvent +} + +func (c *pubSubSubscriptionOnStartHookContext) RequestContext() RequestContext { + return c.requestContext +} + +func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { + return c.subscriptionEventConfiguration +} + +func (c *pubSubSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { + c.events = append(c.events, event) +} + +// EngineEvent is the event used to write to the engine subscription +type EngineEvent struct { + Data []byte +} + +func (e *EngineEvent) GetData() []byte { + return e.Data +} + +type engineSubscriptionOnStartHookContext struct { + requestContext RequestContext + events [][]byte +} + +func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { + return c.requestContext +} + +func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { + c.events = append(c.events, event.GetData()) +} + +func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { + return nil +} + +type SubscriptionOnStartHandler interface { + // OnSubscriptionOnStart is called once at subscription start + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error +} + +// NewPubSubOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a pubsub.OnSubscriptionStartFn +func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) error) datasource.OnSubscriptionStartFn { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { + requestContext := getRequestContext(resolveCtx.Context()) + hookCtx := &pubSubSubscriptionOnStartHookContext{ + requestContext: requestContext, + subscriptionEventConfiguration: subConf, + } + + err := fn(hookCtx) + + return hookCtx.events, err + } +} + +func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) error) graphql_datasource.OnSubscriptionStartFn { + return func(resolveCtx *resolve.Context) ([][]byte, error) { + requestContext := getRequestContext(resolveCtx.Context()) + hookCtx := &engineSubscriptionOnStartHookContext{ + requestContext: requestContext, + } + + err := fn(hookCtx) + + return hookCtx.events, err + } +} From 19ca6fdb61133c2ad8d9153d178ad420cd0a55d5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sat, 19 Jul 2025 16:01:27 +0200 Subject: [PATCH 055/173] chore: update deps --- demo/go.mod | 30 +++------------ demo/go.sum | 90 ++++++--------------------------------------- router-tests/go.mod | 4 +- router/go.mod | 2 +- router/go.sum | 2 + 5 files changed, 22 insertions(+), 106 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 17936f70d8..d1c2b5907c 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,8 +11,8 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router-tests v0.0.0-20250603094938-922424d50778 + github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f + github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.23.1 @@ -24,13 +24,9 @@ require ( require ( connectrpc.com/connect v1.16.2 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/KimMachineGun/automemlimit v0.6.1 // indirect github.com/MicahParks/jwkset v0.9.0 // indirect github.com/MicahParks/keyfunc/v3 v3.3.5 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/benbjohnson/clock v1.3.0 // indirect @@ -44,16 +40,12 @@ require ( github.com/cilium/ebpf v0.16.0 // indirect github.com/cloudflare/backoff v0.0.0-20240920015135-e46b80a3a7d0 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect - github.com/containerd/continuity v0.4.5 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/docker/cli v27.4.1+incompatible // indirect - github.com/docker/docker v27.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dop251/goja v0.0.0-20230906160731-9410bcaa81d2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -74,11 +66,9 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hashicorp/consul/sdk v0.16.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -103,18 +93,11 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.74 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/user v0.3.0 // indirect - github.com/moby/term v0.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/run v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.2.3 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/ory/dockertest/v3 v3.12.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect @@ -151,10 +134,8 @@ require ( github.com/twmb/franz-go/pkg/kmsg v1.7.0 // indirect github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.203 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -183,11 +164,10 @@ require ( golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect google.golang.org/grpc v1.68.1 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rogchap.com/v8go v0.9.0 // indirect ) diff --git a/demo/go.sum b/demo/go.sum index c9e894a7b2..abd0ad07d9 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -1,13 +1,7 @@ connectrpc.com/connect v1.16.2 h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE= connectrpc.com/connect v1.16.2/go.mod h1:n2kgwskMHXC+lVqb18wngEpF95ldBHXjZYJussz5FRc= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/99designs/gqlgen v0.17.76 h1:YsJBcfACWmXWU2t1yCjoGdOmqcTfOFpjbLAE443fmYI= github.com/99designs/gqlgen v0.17.76/go.mod h1:miiU+PkAnTIDKMQ1BseUOIVeQHoiwYDZGCswoxl7xec= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= @@ -15,10 +9,6 @@ github.com/MicahParks/jwkset v0.9.0 h1:xDlGu6mZJdJ+mgAI4mIRqWm2p8Vrx0U98LMgRObw4 github.com/MicahParks/jwkset v0.9.0/go.mod h1:fVrj6TmG1aKlJEeceAz7JsXGTXEn72zP1px3us53JrA= github.com/MicahParks/keyfunc/v3 v3.3.5 h1:7ceAJLUAldnoueHDNzF8Bx06oVcQ5CfJnYwNt1U3YYo= github.com/MicahParks/keyfunc/v3 v3.3.5/go.mod h1:SdCCyMJn/bYqWDvARspC6nCT8Sk74MjuAY22C7dCST8= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= @@ -62,19 +52,15 @@ github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/cloudflare/backoff v0.0.0-20240920015135-e46b80a3a7d0 h1:pRcxfaAlK0vR6nOeQs7eAEvjJzdGXl8+KaBlcvpQTyQ= github.com/cloudflare/backoff v0.0.0-20240920015135-e46b80a3a7d0/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= -github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= -github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= +github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= -github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -91,12 +77,6 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= -github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= @@ -132,8 +112,6 @@ github.com/go-redis/redis_rate/v10 v10.0.1 h1:calPxi7tVlxojKunJwQ72kwfozdy25RjA0 github.com/go-redis/redis_rate/v10 v10.0.1/go.mod h1:EMiuO9+cjRkR7UvdvwMO7vbgqJkltQHtwbdIQvaBKIU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -149,8 +127,6 @@ github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= @@ -164,8 +140,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -208,7 +182,6 @@ github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCX github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/kingledion/go-tools v0.6.0 h1:y8C/4mWoHgLkO45dB+Y/j0o4Y4WUB5lDTAcMPMtFpTg= github.com/kingledion/go-tools v0.6.0/go.mod h1:qcDJQxBui/H/hterGb90GMlLs9Yi7QrwaJL8OGdbsms= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= @@ -227,8 +200,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -253,12 +224,6 @@ github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYC github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= -github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk= @@ -269,16 +234,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= -github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= -github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -390,25 +347,18 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h1:NEUrhuqOaTO1dpW8pz2tu6dKbQAqFvgiF/m4NXdzZm0= github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= -github.com/wundergraph/cosmo/router v0.0.0-20250715110703-10f2e5f9c79e h1:ly42hhxGWNi0Qhv05Or3WwuagBRGD+po7vmL3LYEavY= -github.com/wundergraph/cosmo/router v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:09Zsu2x8iDB6KUf5XkkuRHfgfOYNSCxmVy12/xmhMec= -github.com/wundergraph/cosmo/router-tests v0.0.0-20250603094938-922424d50778 h1:lG3o2OL/vA4REz2qbf/tNtv92k2QQasQUcVE9o1rsO0= -github.com/wundergraph/cosmo/router-tests v0.0.0-20250603094938-922424d50778/go.mod h1:drI7pUgQzLdqLJRSq+R5Gzqim3tIzYk2M+NRk5Zk4dc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.203 h1:qTMYS9EICDCoMY90ILE3eW2/i1VNMhmyl79qpw5v6xc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.203/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= +github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= +github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f h1:ZxrYipVu20RKNWAMduseyExlZKl7uOXoNsIFtxQrcwo= +github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f/go.mod h1:2ORZCPYHJS5IqKnJztnPkMzyPf2U+NV6gi4JAYOOG80= +github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= +github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 h1:g2MpMjU/Jk30oBzfBjGRgH3EzTvwI0IV57HhlUjeyZc= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= @@ -466,8 +416,6 @@ go.withmatt.com/connect-brotli v0.4.0 h1:7ObWkYmEbUXK3EKglD0Lgj0BBnnD3jNdAxeDRct go.withmatt.com/connect-brotli v0.4.0/go.mod h1:c2eELz56za+/Mxh1yJrlglZ4VM9krpOCPqS2Vxf8NVk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= @@ -475,8 +423,6 @@ golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0J golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= @@ -484,15 +430,11 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= @@ -502,11 +444,9 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -538,21 +478,17 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0= gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU= google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= @@ -570,8 +506,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rogchap.com/v8go v0.9.0 h1:wYbUCO4h6fjTamziHrzyrPnpFNuzPpjZY+nfmZjNaew= rogchap.com/v8go v0.9.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs= diff --git a/router-tests/go.mod b/router-tests/go.mod index 56e8f747a2..6ff4161972 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -23,9 +23,9 @@ require ( github.com/twmb/franz-go v1.16.1 github.com/twmb/franz-go/pkg/kadm v1.11.0 github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 - github.com/wundergraph/cosmo/demo v0.0.0-20250715133706-4c418b758ddd + github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250715133706-4c418b758ddd + github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 diff --git a/router/go.mod b/router/go.mod index 79ef6d2a60..2c17b1e8eb 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 74707730d5..10be666c36 100644 --- a/router/go.sum +++ b/router/go.sum @@ -292,6 +292,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 h1:g2MpMjU/Jk30oBzfBjGRgH3EzTvwI0IV57HhlUjeyZc= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 h1:ZZHwuFB/vQp0hdo4rySJ5wLrBCGziV8rcMpi2dDRUTI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 79f05e5ea6fd4e5fc2e764031cabd098b2711920 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sat, 19 Jul 2025 16:04:56 +0200 Subject: [PATCH 056/173] chore: go mod tidy --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.sum | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 6ff4161972..3d7fa436e0 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 995b25670e..9c4bf55a97 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 h1:g2MpMjU/Jk30oBzfBjGRgH3EzTvwI0IV57HhlUjeyZc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 h1:ZZHwuFB/vQp0hdo4rySJ5wLrBCGziV8rcMpi2dDRUTI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.sum b/router/go.sum index 10be666c36..f962603a28 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,6 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 h1:g2MpMjU/Jk30oBzfBjGRgH3EzTvwI0IV57HhlUjeyZc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 h1:ZZHwuFB/vQp0hdo4rySJ5wLrBCGziV8rcMpi2dDRUTI= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 4b9c8a738b634a49a8022938595fd9bb80bb40da Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Sat, 19 Jul 2025 20:04:55 +0200 Subject: [PATCH 057/173] fix: properly escapes values --- router/pkg/pubsub/kafka/engine_datasource.go | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index b7a9da1d45..60b4fb40f9 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -81,7 +81,27 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { return "", err } - return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil + // Properly escape string values for JSON + topicBytes, err := json.Marshal(s.Topic) + if err != nil { + return "", err + } + + key := s.Event.Key + if key == nil { + key = []byte{} + } + keyBytes, err := json.Marshal(string(key)) + if err != nil { + return "", err + } + + providerBytes, err := json.Marshal(s.ProviderID()) + if err != nil { + return "", err + } + + return fmt.Sprintf(`{"topic":%s, "event": {"data": %s, "key": %s, "headers": %s}, "providerId":%s}`, topicBytes, s.Event.Data, keyBytes, headersBytes, providerBytes), nil } type SubscriptionDataSource struct { From 92575659e89267ef6f28a380e5c6585cf8a774e1 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 21 Jul 2025 12:51:41 +0200 Subject: [PATCH 058/173] chore: align MarshalJSONTemplate and improve the kafka one --- router/pkg/pubsub/kafka/engine_datasource.go | 22 +------------------ router/pkg/pubsub/nats/engine_datasource.go | 4 ++-- .../pubsub/nats/engine_datasource_factory.go | 2 +- .../pkg/pubsub/nats/engine_datasource_test.go | 3 ++- 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 60b4fb40f9..908e3757f8 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -81,27 +81,7 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { return "", err } - // Properly escape string values for JSON - topicBytes, err := json.Marshal(s.Topic) - if err != nil { - return "", err - } - - key := s.Event.Key - if key == nil { - key = []byte{} - } - keyBytes, err := json.Marshal(string(key)) - if err != nil { - return "", err - } - - providerBytes, err := json.Marshal(s.ProviderID()) - if err != nil { - return "", err - } - - return fmt.Sprintf(`{"topic":%s, "event": {"data": %s, "key": %s, "headers": %s}, "providerId":%s}`, topicBytes, s.Event.Data, keyBytes, headersBytes, providerBytes), nil + return fmt.Sprintf(`{"topic": "%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId": "%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil } type SubscriptionDataSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index c3f910dd0e..72779273bd 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -74,10 +74,10 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() string { +func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() (string, error) { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()) + return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()), nil } type SubscriptionSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index f37c1dc8cc..e43c49e8ea 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -69,7 +69,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri FieldName: c.fieldName, } - return evtCfg.MarshalJSONTemplate(), nil + return evtCfg.MarshalJSONTemplate() } func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 5213b21263..79b8219e53 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -44,7 +44,8 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := tt.config.MarshalJSONTemplate() + result, err := tt.config.MarshalJSONTemplate() + assert.NoError(t, err) assert.Equal(t, tt.wantPattern, result) }) } From d6c6ec69cb61e0f70340eea6e259bd22ea0dd4e1 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 21 Jul 2025 13:38:07 +0200 Subject: [PATCH 059/173] chore: small fix to MarshalJSONTemplate format --- router/pkg/pubsub/kafka/engine_datasource.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 908e3757f8..b7a9da1d45 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -81,7 +81,7 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { return "", err } - return fmt.Sprintf(`{"topic": "%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId": "%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil + return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil } type SubscriptionDataSource struct { From a8b7f7d94ebe8720833e1f4e9edae2769fdc07f8 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 22 Jul 2025 12:24:45 +0200 Subject: [PATCH 060/173] chore: use the new subscription hooks on the engine --- router-tests/modules/start_subscription_test.go | 2 +- router/core/subscriptions_modules.go | 2 +- router/go.mod | 2 +- router/go.sum | 4 ++-- router/pkg/pubsub/datasource/hookeddatasource.go | 12 ++++++++---- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 1557dfad9a..4b32ed2019 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -263,7 +263,7 @@ func TestStartSubscriptionHook(t *testing.T) { }, "unable to close client before timeout") requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") - assert.Len(t, requestLog.All(), 1) + assert.Len(t, requestLog.All(), 2) t.Cleanup(func() { require.Len(t, subscriptionOneArgsCh, 0) }) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 86b74d2d81..8e3cfd44bd 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -91,7 +91,7 @@ func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext } func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) error) graphql_datasource.OnSubscriptionStartFn { - return func(resolveCtx *resolve.Context) ([][]byte, error) { + return func(resolveCtx *resolve.Context, input []byte) ([][]byte, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, diff --git a/router/go.mod b/router/go.mod index 2c17b1e8eb..4d036ed172 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index f962603a28..1a5b845d58 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 h1:ZZHwuFB/vQp0hdo4rySJ5wLrBCGziV8rcMpi2dDRUTI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1 h1:33Y5dYwaby/F7TziB+xl5Ma9m8Z4U8MMhHKkl6dOWG4= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 6f24e1aab8..7855f91b1e 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -10,18 +10,22 @@ type HookedSubscriptionDataSource struct { SubscriptionDataSource PubSubSubscriptionDataSource } -func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - subscriptionEventUpdater := NewSubscriptionEventUpdater(updater) +func (h *HookedSubscriptionDataSource) OnSubscriptionStart(ctx *resolve.Context, input []byte) (err error) { for _, fn := range h.OnSubscriptionStartFns { events, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil { return err } for _, event := range events { - subscriptionEventUpdater.Update(event) + ctx.EmitSubscriptionUpdate(event.GetData()) } } - return h.SubscriptionDataSource.Start(ctx, input, subscriptionEventUpdater) + + return nil +} + +func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { + return h.SubscriptionDataSource.Start(ctx, input, NewSubscriptionEventUpdater(updater)) } func (h *HookedSubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) { From 18bee17e296427190febac2efd60d9108ea70ff6 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 22 Jul 2025 12:37:00 +0200 Subject: [PATCH 061/173] chore: update engine version --- router/go.mod | 2 +- router/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/go.mod b/router/go.mod index 4d036ed172..7c9c08fe9a 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 1a5b845d58..10cfd54f8c 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1 h1:33Y5dYwaby/F7TziB+xl5Ma9m8Z4U8MMhHKkl6dOWG4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722101213-2a6b5f067bf1/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb h1:g/btv86ug7RfYxSbSVjkvC8WuP4wpVOb/PpavJj4Jd8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From bda2ed628c392e386ac9be6b71da70ca750b0e05 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 22 Jul 2025 12:37:43 +0200 Subject: [PATCH 062/173] chore: tidy mods of router-tests --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 3d7fa436e0..bcb8fe2bc7 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 9c4bf55a97..5d2d31905b 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84 h1:ZZHwuFB/vQp0hdo4rySJ5wLrBCGziV8rcMpi2dDRUTI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250719113655-c970c079ff84/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb h1:g/btv86ug7RfYxSbSVjkvC8WuP4wpVOb/PpavJj4Jd8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 638c6f72ea155f066b1671d0bbb3ec5c2731c21d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 00:32:48 +0200 Subject: [PATCH 063/173] chore: hooks can close subscription and specify custom errors --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 +- .../modules/start-subscription/module.go | 6 +- .../modules/start_subscription_test.go | 184 +++++++++++++++++- router/core/errors.go | 5 + router/core/graphql_handler.go | 16 ++ router/core/router_config.go | 2 +- router/core/subscriptions_modules.go | 57 ++++-- router/go.mod | 2 +- router/go.sum | 4 +- .../pkg/pubsub/datasource/hookeddatasource.go | 12 +- router/pkg/pubsub/datasource/provider.go | 2 +- 12 files changed, 258 insertions(+), 38 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index bcb8fe2bc7..0c0825e7c9 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 5d2d31905b..7f1d9cb02c 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb h1:g/btv86ug7RfYxSbSVjkvC8WuP4wpVOb/PpavJj4Jd8= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d h1:8VUZhQOlatGXwbj67FQAZWX/h34it3IjGr0ajceltMs= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go index 6ece70d09c..60d4dc652c 100644 --- a/router-tests/modules/start-subscription/module.go +++ b/router-tests/modules/start-subscription/module.go @@ -10,7 +10,7 @@ const myModuleID = "startSubscriptionModule" type StartSubscriptionModule struct { Logger *zap.Logger - Callback func(ctx core.SubscriptionOnStartHookContext) error + Callback func(ctx core.SubscriptionOnStartHookContext) (bool, error) } func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { @@ -20,7 +20,7 @@ func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { return nil } -func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { +func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) (bool, error) { m.Logger.Info("SubscriptionOnStart Hook has been run") @@ -28,7 +28,7 @@ func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnSta return m.Callback(ctx) } - return nil + return false, nil } func (m *StartSubscriptionModule) Module() core.ModuleInfo { diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 4b32ed2019..ff0b6013d2 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -2,6 +2,7 @@ package module_test import ( "errors" + "net/http" "testing" "time" @@ -89,12 +90,12 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) error { + Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { ctx.WriteEvent(&kafka.Event{ Key: []byte("1"), Data: []byte(`{"id": 1, "__typename": "Employee"}`), }) - return nil + return false, nil }, }, }, @@ -166,6 +167,88 @@ func TestStartSubscriptionHook(t *testing.T) { }) }) + t.Run("Test StartSubscription with close to true", func(t *testing.T) { + t.Parallel() + + callbackCalled := make(chan bool) + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + callbackCalled <- true + return true, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: $employeeID)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "employeeID": 3, + } + type kafkaSubscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionArgsCh := make(chan kafkaSubscriptionArgs, 1) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, vars, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + <-callbackCalled + xEnv.WaitForSubscriptionCount(0, time.Second*10) + + // require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + + require.Len(t, subscriptionArgsCh, 0) + }) + }) + t.Run("Test StartSubscription write event sends event only to the subscription", func(t *testing.T) { t.Parallel() @@ -173,15 +256,15 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) error { + Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { employeeId := ctx.RequestContext().Operation().Variables().GetInt64("employeeID") if employeeId != 1 { - return nil + return false, nil } ctx.WriteEvent(&kafka.Event{ Data: []byte(`{"id": 1, "__typename": "Employee"}`), }) - return nil + return false, nil }, }, }, @@ -270,6 +353,93 @@ func TestStartSubscriptionHook(t *testing.T) { }) }) + t.Run("Test StartSubscription error is propagated to the client", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + return false, core.NewCustomModuleError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected)) + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscription struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: $employeeID)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "employeeID": 1, + } + type kafkaSubscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionOneArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscription, vars, func(dataValue []byte, errValue error) error { + subscriptionOneArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, time.Second*10) + + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + var graphqlErrs graphql.Errors + require.ErrorAs(t, args.errValue, &graphqlErrs) + statusCode, ok := graphqlErrs[0].Extensions["statusCode"].(float64) + require.True(t, ok, "statusCode is not a float64") + require.Equal(t, http.StatusLoopDetected, int(statusCode)) + require.Equal(t, http.StatusText(http.StatusLoopDetected), graphqlErrs[0].Extensions["code"]) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + t.Cleanup(func() { + require.Len(t, subscriptionOneArgsCh, 0) + }) + }) + }) + t.Run("Test StartSubscription hook is called for engine subscription", func(t *testing.T) { t.Parallel() @@ -333,11 +503,11 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) error { + Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { ctx.WriteEvent(&core.EngineEvent{ Data: []byte(`{"data":{"countEmp":1000}}`), }) - return nil + return false, nil }, }, }, diff --git a/router/core/errors.go b/router/core/errors.go index 7f8df34da2..070463ca58 100644 --- a/router/core/errors.go +++ b/router/core/errors.go @@ -35,6 +35,7 @@ const ( errorTypeInvalidWsSubprotocol errorTypeEDFSInvalidMessage errorTypeMergeResult + errorTypeCustomModuleError ) type ( @@ -89,6 +90,10 @@ func getErrorType(err error) errorType { if errors.As(err, &mergeResultErr) { return errorTypeMergeResult } + var customModuleErr *CustomModuleError + if errors.As(err, &customModuleErr) { + return errorTypeCustomModuleError + } return errorTypeUnknown } diff --git a/router/core/graphql_handler.go b/router/core/graphql_handler.go index c494fff4ce..921de2e5ed 100644 --- a/router/core/graphql_handler.go +++ b/router/core/graphql_handler.go @@ -400,6 +400,22 @@ func (h *GraphQLHandler) WriteError(ctx *resolve.Context, err error, res *resolv if isHttpResponseWriter { httpWriter.WriteHeader(http.StatusInternalServerError) } + case errorTypeCustomModuleError: + var customModuleErr *CustomModuleError + if !errors.As(err, &customModuleErr) { + response.Errors[0].Message = "Internal server error" + return + } + response.Errors[0].Message = customModuleErr.Message() + if customModuleErr.Code() != "" || customModuleErr.StatusCode() != 0 { + response.Errors[0].Extensions = &Extensions{ + Code: customModuleErr.Code(), + StatusCode: customModuleErr.StatusCode(), + } + } + if isHttpResponseWriter { + httpWriter.WriteHeader(customModuleErr.StatusCode()) + } } if ctx.TracingOptions.Enable && ctx.TracingOptions.IncludeTraceOutputInResponseExtensions { diff --git a/router/core/router_config.go b/router/core/router_config.go index b97ba5eecb..1a33bb105b 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -26,7 +26,7 @@ import ( ) type subscriptionHooks struct { - startSubscription []func(ctx SubscriptionOnStartHookContext) error + startSubscription []func(ctx SubscriptionOnStartHookContext) (bool, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 8e3cfd44bd..aec7bb48da 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -6,14 +6,37 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -// SubscriptionHookError is used to customize the error messages and the behavior -type SubscriptionHookError struct { - HttpError HttpError - CloseSubscription bool +// CustomModuleError is used to customize the error messages and the behavior +type CustomModuleError struct { + err error + message string + statusCode int + code string } -func (e *SubscriptionHookError) Error() string { - return e.HttpError.Error() +func (e *CustomModuleError) Error() string { + return e.err.Error() +} + +func (e *CustomModuleError) Message() string { + return e.message +} + +func (e *CustomModuleError) StatusCode() int { + return e.statusCode +} + +func (e *CustomModuleError) Code() string { + return e.code +} + +func NewCustomModuleError(err error, message string, statusCode int, code string) *CustomModuleError { + return &CustomModuleError{ + err: err, + message: message, + statusCode: statusCode, + code: code, + } } type SubscriptionOnStartHookContext interface { @@ -71,34 +94,36 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() type SubscriptionOnStartHandler interface { // OnSubscriptionOnStart is called once at subscription start - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error + // If the boolean is true, the subscription is closed. + // The error is propagated to the client. + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) (bool, error) } // NewPubSubOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a pubsub.OnSubscriptionStartFn -func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) error) datasource.OnSubscriptionStartFn { - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, error) { +func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.OnSubscriptionStartFn { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &pubSubSubscriptionOnStartHookContext{ requestContext: requestContext, subscriptionEventConfiguration: subConf, } - err := fn(hookCtx) + close, err := fn(hookCtx) - return hookCtx.events, err + return hookCtx.events, close, err } } -func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) error) graphql_datasource.OnSubscriptionStartFn { - return func(resolveCtx *resolve.Context, input []byte) ([][]byte, error) { +// NewEngineOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.OnSubscriptionStartFn +func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.OnSubscriptionStartFn { + return func(resolveCtx *resolve.Context, input []byte) ([][]byte, bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, } - err := fn(hookCtx) + close, err := fn(hookCtx) - return hookCtx.events, err + return hookCtx.events, close, err } } diff --git a/router/go.mod b/router/go.mod index 7c9c08fe9a..6d9b7f75d8 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 10cfd54f8c..8cb233eb7c 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb h1:g/btv86ug7RfYxSbSVjkvC8WuP4wpVOb/PpavJj4Jd8= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722103421-1e6ed5fe1acb/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d h1:8VUZhQOlatGXwbj67FQAZWX/h34it3IjGr0ajceltMs= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 7855f91b1e..611f0c2140 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -10,18 +10,22 @@ type HookedSubscriptionDataSource struct { SubscriptionDataSource PubSubSubscriptionDataSource } -func (h *HookedSubscriptionDataSource) OnSubscriptionStart(ctx *resolve.Context, input []byte) (err error) { +func (h *HookedSubscriptionDataSource) OnSubscriptionStart(ctx *resolve.Context, input []byte) (close bool, err error) { for _, fn := range h.OnSubscriptionStartFns { - events, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) + events, close, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil { - return err + return close, err } for _, event := range events { ctx.EmitSubscriptionUpdate(event.GetData()) } + // if close is true, the subscription should be close, so there is no need to call the next hook + if close { + return true, nil + } } - return nil + return false, nil } func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index a19c0eb937..92df81794f 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -50,7 +50,7 @@ type StreamEvent interface { GetData() []byte } -type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) ([]StreamEvent, error) +type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) ([]StreamEvent, bool, error) // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { From cf1c868645ae348dd4d2c418facb0ae1b03e4344 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 09:44:12 +0200 Subject: [PATCH 064/173] chore: WriteEvents now calls directly resolveCtx.EmitSubscriptionUpdate --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/subscriptions_modules.go | 18 ++++++++++-------- router/go.mod | 2 +- router/go.sum | 4 ++-- .../pkg/pubsub/datasource/hookeddatasource.go | 15 ++++----------- router/pkg/pubsub/datasource/provider.go | 2 +- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 0c0825e7c9..199a7cad11 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 7f1d9cb02c..3e0cdfa07f 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d h1:8VUZhQOlatGXwbj67FQAZWX/h34it3IjGr0ajceltMs= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c h1:30qciAytUG09OlG3nZja4ZNBvJ3LW9nyAImlOlryY8g= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index aec7bb48da..fae9426638 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -51,7 +51,7 @@ type SubscriptionOnStartHookContext interface { type pubSubSubscriptionOnStartHookContext struct { requestContext RequestContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration - events []datasource.StreamEvent + writeEventHook func(data []byte) } func (c *pubSubSubscriptionOnStartHookContext) RequestContext() RequestContext { @@ -63,7 +63,7 @@ func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() } func (c *pubSubSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - c.events = append(c.events, event) + c.writeEventHook(event.GetData()) } // EngineEvent is the event used to write to the engine subscription @@ -77,7 +77,7 @@ func (e *EngineEvent) GetData() []byte { type engineSubscriptionOnStartHookContext struct { requestContext RequestContext - events [][]byte + writeEventHook func(data []byte) } func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { @@ -85,7 +85,7 @@ func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { } func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - c.events = append(c.events, event.GetData()) + c.writeEventHook(event.GetData()) } func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { @@ -101,29 +101,31 @@ type SubscriptionOnStartHandler interface { // NewPubSubOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a pubsub.OnSubscriptionStartFn func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.OnSubscriptionStartFn { - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) ([]datasource.StreamEvent, bool, error) { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &pubSubSubscriptionOnStartHookContext{ requestContext: requestContext, subscriptionEventConfiguration: subConf, + writeEventHook: resolveCtx.EmitSubscriptionUpdate, } close, err := fn(hookCtx) - return hookCtx.events, close, err + return close, err } } // NewEngineOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.OnSubscriptionStartFn func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.OnSubscriptionStartFn { - return func(resolveCtx *resolve.Context, input []byte) ([][]byte, bool, error) { + return func(resolveCtx *resolve.Context, input []byte) (bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, + writeEventHook: resolveCtx.EmitSubscriptionUpdate, } close, err := fn(hookCtx) - return hookCtx.events, close, err + return close, err } } diff --git a/router/go.mod b/router/go.mod index 6d9b7f75d8..85485368e4 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 8cb233eb7c..7ca720d1a4 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d h1:8VUZhQOlatGXwbj67FQAZWX/h34it3IjGr0ajceltMs= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250722223039-2f343271b03d/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c h1:30qciAytUG09OlG3nZja4ZNBvJ3LW9nyAImlOlryY8g= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 611f0c2140..7078839752 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -12,20 +12,13 @@ type HookedSubscriptionDataSource struct { func (h *HookedSubscriptionDataSource) OnSubscriptionStart(ctx *resolve.Context, input []byte) (close bool, err error) { for _, fn := range h.OnSubscriptionStartFns { - events, close, err := fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) - if err != nil { - return close, err - } - for _, event := range events { - ctx.EmitSubscriptionUpdate(event.GetData()) - } - // if close is true, the subscription should be close, so there is no need to call the next hook - if close { - return true, nil + close, err = fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) + if err != nil || close { + return } } - return false, nil + return } func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 92df81794f..bd6f6cbc81 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -50,7 +50,7 @@ type StreamEvent interface { GetData() []byte } -type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) ([]StreamEvent, bool, error) +type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { From 89832b9a120cc0a5a2110c3fcccdd6a0e275f61f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 10:11:40 +0200 Subject: [PATCH 065/173] fix: minor typos on adr --- adr/cosmo-streams-v1.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index f02729dfa6..d5929b8538 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -160,8 +160,8 @@ import ( ) func init() { - // Register your module here and it will be loaded at router start - core.RegisterModule(&MyModule{}) + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) } type MyModule struct {} @@ -255,7 +255,7 @@ func (m *MyModule) Module() core.ModuleInfo { // Interface guards var ( - _ core.StreamBatchEventHook = (*MyModule)(nil) + _ core.StreamBatchEventHook = (*MyModule)(nil) ) ``` @@ -304,8 +304,8 @@ import ( ) func init() { - // Register your module here and it will be loaded at router start - core.RegisterModule(&MyModule{}) + // Register your module here and it will be loaded at router start + core.RegisterModule(&MyModule{}) } type MyModule struct {} @@ -366,7 +366,7 @@ func (m *MyModule) Module() core.ModuleInfo { // Interface guards var ( - _ core.StreamBatchEventHook = (*MyModule)(nil) + _ core.SubscriptionOnStartHandler = (*MyModule)(nil) ) ``` From 7b8239f763c7e6e61cedd759a0f75af492b9d0b4 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 11:42:14 +0200 Subject: [PATCH 066/173] chore: aligned names in the engine with the ones here --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/factoryresolver.go | 4 ++-- router/core/subscriptions_modules.go | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 015a387602..7d92ff6209 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index b8b9070916..069efed354 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c h1:30qciAytUG09OlG3nZja4ZNBvJ3LW9nyAImlOlryY8g= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 h1:/wIbOVZfFduj2rS9TYHqBiTrBcXkRcqqWHTTtLSngMw= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index d07733ae39..8ca77ba619 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -415,7 +415,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } - onSubscriptionStarts := make([]graphql_datasource.OnSubscriptionStartFn, len(l.subscriptionHooks.startSubscription)) + onSubscriptionStarts := make([]graphql_datasource.SubscriptionOnStartFn, len(l.subscriptionHooks.startSubscription)) for i, fn := range l.subscriptionHooks.startSubscription { onSubscriptionStarts[i] = NewEngineOnSubscriptionStartHook(fn) } @@ -432,7 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - OnSubscriptionStartFns: onSubscriptionStarts, + SubscriptionOnStartFns: onSubscriptionStarts, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index fae9426638..d8d0d53fd4 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -93,7 +93,7 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() } type SubscriptionOnStartHandler interface { - // OnSubscriptionOnStart is called once at subscription start + // SubscriptionOnStart is called once at subscription start // If the boolean is true, the subscription is closed. // The error is propagated to the client. SubscriptionOnStart(ctx SubscriptionOnStartHookContext) (bool, error) @@ -116,7 +116,7 @@ func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext } // NewEngineOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.OnSubscriptionStartFn -func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.OnSubscriptionStartFn { +func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.SubscriptionOnStartFn { return func(resolveCtx *resolve.Context, input []byte) (bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ diff --git a/router/go.mod b/router/go.mod index c7be42f169..2835baec49 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 7407a38fa8..9a6fce8f5f 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c h1:30qciAytUG09OlG3nZja4ZNBvJ3LW9nyAImlOlryY8g= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207.0.20250723073946-dc89310fee4c/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 h1:/wIbOVZfFduj2rS9TYHqBiTrBcXkRcqqWHTTtLSngMw= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 9c8609f3a280837b04a7e24b7f049d1846fb8103 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 15:57:12 +0200 Subject: [PATCH 067/173] chore: use new engine hooks --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- router/pkg/pubsub/datasource/hookeddatasource.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 7d92ff6209..5b79c81700 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 069efed354..f069674e1d 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 h1:/wIbOVZfFduj2rS9TYHqBiTrBcXkRcqqWHTTtLSngMw= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b h1:kVjcnkdcAtt3XxT/bVogWR3HA8K6lCOGEroZYTvkbys= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 2835baec49..500b5c447c 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 9a6fce8f5f..9ef166774b 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1 h1:/wIbOVZfFduj2rS9TYHqBiTrBcXkRcqqWHTTtLSngMw= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723094008-91dbadd9d6b1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b h1:kVjcnkdcAtt3XxT/bVogWR3HA8K6lCOGEroZYTvkbys= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index 7078839752..dec868aaa7 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -10,7 +10,7 @@ type HookedSubscriptionDataSource struct { SubscriptionDataSource PubSubSubscriptionDataSource } -func (h *HookedSubscriptionDataSource) OnSubscriptionStart(ctx *resolve.Context, input []byte) (close bool, err error) { +func (h *HookedSubscriptionDataSource) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { for _, fn := range h.OnSubscriptionStartFns { close, err = fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil || close { From d6dd642c326f5c852c5146b07ce8b9c39effaced Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 17:31:32 +0200 Subject: [PATCH 068/173] chore: update engine --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 5b79c81700..b4a98cd934 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index f069674e1d..e29ca64cc1 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b h1:kVjcnkdcAtt3XxT/bVogWR3HA8K6lCOGEroZYTvkbys= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 h1:V0se42mIRYGNH+Uj1YgC8J3Jgpneson2wmhKF/EyaOc= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 500b5c447c..0257277d37 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 9ef166774b..3f12d6eaef 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b h1:kVjcnkdcAtt3XxT/bVogWR3HA8K6lCOGEroZYTvkbys= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723135227-207adf74f96b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 h1:V0se42mIRYGNH+Uj1YgC8J3Jgpneson2wmhKF/EyaOc= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 6ad6f59ccafd64d06a52f08bd96d547824fb5279 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 17:41:27 +0200 Subject: [PATCH 069/173] chore: update engine --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index b4a98cd934..97796f26a8 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index e29ca64cc1..193b125483 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 h1:V0se42mIRYGNH+Uj1YgC8J3Jgpneson2wmhKF/EyaOc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c h1:3RAjYdKjtRegTrKkeqwQmrxWsoeyS27oYkjBeKygo/c= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 0257277d37..20b5a63647 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 3f12d6eaef..d17278fd8a 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498 h1:V0se42mIRYGNH+Uj1YgC8J3Jgpneson2wmhKF/EyaOc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153011-3257c5285498/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c h1:3RAjYdKjtRegTrKkeqwQmrxWsoeyS27oYkjBeKygo/c= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From c4dd131433adbb4e78870a187a4e1e1e3c16b666 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 17:55:23 +0200 Subject: [PATCH 070/173] fix: make CustomModuleError panic-free --- router/core/subscriptions_modules.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index d8d0d53fd4..1c31a78b5d 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -15,7 +15,10 @@ type CustomModuleError struct { } func (e *CustomModuleError) Error() string { - return e.err.Error() + if e.err != nil { + return e.err.Error() + } + return e.message } func (e *CustomModuleError) Message() string { From 2e737dae78af291ef4fb9fdbcec8f15f95612cb6 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 17:57:34 +0200 Subject: [PATCH 071/173] fix: make NewPubSubOnSubscriptionStartHook and NewEngineOnSubscriptionStartHook panics free --- router/core/subscriptions_modules.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 1c31a78b5d..ce28b80da2 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -104,6 +104,10 @@ type SubscriptionOnStartHandler interface { // NewPubSubOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a pubsub.OnSubscriptionStartFn func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.OnSubscriptionStartFn { + if fn == nil { + return nil + } + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &pubSubSubscriptionOnStartHookContext{ @@ -120,6 +124,10 @@ func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext // NewEngineOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.OnSubscriptionStartFn func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.SubscriptionOnStartFn { + if fn == nil { + return nil + } + return func(resolveCtx *resolve.Context, input []byte) (bool, error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ From 6ea1ee1a710e3e192bc41c2e4f72121bc9c2bf36 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 17:59:24 +0200 Subject: [PATCH 072/173] chore: clarify SubscriptionOnStartHookContext interface --- router/core/subscriptions_modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index ce28b80da2..68fe78b6c5 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -45,7 +45,7 @@ func NewCustomModuleError(err error, message string, statusCode int, code string type SubscriptionOnStartHookContext interface { // the request context RequestContext() RequestContext - // the subscription event configuration + // the subscription event configuration (will return nil for engine subscription) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration // write an event to the stream of the current subscription WriteEvent(event datasource.StreamEvent) From 057a01f9857de782a7d7a1143554cea6e9fea521 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 23 Jul 2025 19:19:22 +0200 Subject: [PATCH 073/173] chore: small coherency fixes --- rfc/cosmo-streams-v1.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rfc/cosmo-streams-v1.md b/rfc/cosmo-streams-v1.md index dd76c2e29e..2a7cd761f1 100644 --- a/rfc/cosmo-streams-v1.md +++ b/rfc/cosmo-streams-v1.md @@ -99,9 +99,9 @@ func (m *MyModule) Module() core.ModuleInfo { Add a new hook to the subscription lifecycle, `SubscriptionOnStartHandler`, that will be called once at subscription start. The hook arguments are: -* `ctx SubscriptionContext`: The subscription context, which contains the request context and, optionally, the stream context +* `ctx SubscriptionOnStartHookContext`: The subscription context, which contains the request context and, optionally, the subscription event configuration, and a method to emit the event to the stream -`RequestContext` already exists and requires no changes, but `SubscriptionContext` is new. +`RequestContext` already exists and requires no changes, but `SubscriptionEventConfiguration` is new. The hook should return an error if the client is not allowed to subscribe to the stream, preventing the subscription from starting. The hook should return `nil` if the client is allowed to subscribe to the stream, allowing the subscription to proceed. @@ -110,7 +110,8 @@ The hook can return a `SubscriptionHookError` to customize the error messages an I evaluated the possibility of adding the `SubscriptionContext` to the request context and using it within one of the existing hooks, but it would be difficult to build the subscription context without executing the pubsub code. -The `StreamContext.SubscriptionEventConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. +The `SubscriptionEventConfiguration()` contains the subscription configuration as used by the provider. This allows the hooks system to be provider-agnostic, so adding a new provider will not require changes to the hooks system. To use specific fields, the hook can cast the configuration to the specific type for the provider. +The `WriteEvent()` method is new and allows the hook to emit the event to the stream. ## Initial Message @@ -163,7 +164,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error }, } // emit the event to the stream, that will be received only by the client that subscribed to the stream - ctx.StreamContext().WriteEvent(evt) + ctx.WriteEvent(evt) } return nil } @@ -616,12 +617,12 @@ type MyModule struct {} func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { // check if the provider is nats - if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } // check if the provider id is the one expected by the module - if ctx.StreamContext().ProviderID() != "my-nats" { + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-nats" { return events, nil } @@ -781,7 +782,6 @@ type StreamPublishEventHookContext interface { type SubscriptionOnStartHookContext interface { RequestContext() RequestContext - StreamContext() StreamContext SubscriptionEventConfiguration() SubscriptionEventConfiguration WriteEvent(event core.StreamEvent) } @@ -1005,11 +1005,11 @@ import ( type MyModule struct {} func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { - if ctx.StreamContext().ProviderType() != "nats" { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } - if ctx.StreamContext().ProviderID() != "my-nats" { + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-nats" { return events, nil } From 73292a7a41d5d6d0d2e3e50eac6e83dd4c9e5acf Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 24 Jul 2025 10:54:24 +0200 Subject: [PATCH 074/173] chore: small fixes and made names more coherent --- router-tests/modules/start_subscription_test.go | 1 - router/core/factoryresolver.go | 16 ++++++++-------- router/core/router.go | 2 +- router/core/router_config.go | 2 +- router/core/subscriptions_modules.go | 8 ++++---- router/pkg/pubsub/datasource/factory.go | 6 +++--- router/pkg/pubsub/datasource/hookeddatasource.go | 4 ++-- router/pkg/pubsub/datasource/planner.go | 2 +- router/pkg/pubsub/datasource/provider.go | 2 +- router/pkg/pubsub/kafka/engine_datasource.go | 4 ++++ router/pkg/pubsub/nats/engine_datasource.go | 4 ++++ router/pkg/pubsub/pubsub.go | 4 ++-- router/pkg/pubsub/redis/engine_datasource.go | 4 ++++ 13 files changed, 35 insertions(+), 24 deletions(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index ff0b6013d2..e23c75ca5f 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -236,7 +236,6 @@ func TestStartSubscriptionHook(t *testing.T) { <-callbackCalled xEnv.WaitForSubscriptionCount(0, time.Second*10) - // require.NoError(t, client.Close()) testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { require.NoError(t, err) diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 8ca77ba619..6827834db4 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -415,9 +415,9 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } - onSubscriptionStarts := make([]graphql_datasource.SubscriptionOnStartFn, len(l.subscriptionHooks.startSubscription)) - for i, fn := range l.subscriptionHooks.startSubscription { - onSubscriptionStarts[i] = NewEngineOnSubscriptionStartHook(fn) + subscriptionOnStartFns := make([]graphql_datasource.SubscriptionOnStartFn, len(l.subscriptionHooks.onStart)) + for i, fn := range l.subscriptionHooks.onStart { + subscriptionOnStartFns[i] = NewEngineSubscriptionOnStartHook(fn) } customConfiguration, err := graphql_datasource.NewConfiguration(graphql_datasource.ConfigurationInput{ Fetch: &graphql_datasource.FetchConfiguration{ @@ -432,7 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - SubscriptionOnStartFns: onSubscriptionStarts, + SubscriptionOnStartFns: subscriptionOnStartFns, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, @@ -474,9 +474,9 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod } } - onSubscriptionStarts := make([]pubsub_datasource.OnSubscriptionStartFn, len(l.subscriptionHooks.startSubscription)) - for i, fn := range l.subscriptionHooks.startSubscription { - onSubscriptionStarts[i] = NewPubSubOnSubscriptionStartHook(fn) + subscriptionOnStartFns := make([]pubsub_datasource.SubscriptionOnStartFn, len(l.subscriptionHooks.onStart)) + for i, fn := range l.subscriptionHooks.onStart { + subscriptionOnStartFns[i] = NewPubSubSubscriptionOnStartHook(fn) } factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, @@ -486,7 +486,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod l.resolver.InstanceData().HostName, l.resolver.InstanceData().ListenAddress, pubsub.Hooks{ - OnSubscriptionStarts: onSubscriptionStarts, + SubscriptionOnStart: subscriptionOnStartFns, }, ) if err != nil { diff --git a/router/core/router.go b/router/core/router.go index 2f83d284e1..1d1414470a 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -643,7 +643,7 @@ func (r *Router) initModules(ctx context.Context) error { } if handler, ok := moduleInstance.(SubscriptionOnStartHandler); ok { - r.subscriptionHooks.startSubscription = append(r.subscriptionHooks.startSubscription, handler.SubscriptionOnStart) + r.subscriptionHooks.onStart = append(r.subscriptionHooks.onStart, handler.SubscriptionOnStart) } r.modules = append(r.modules, moduleInstance) diff --git a/router/core/router_config.go b/router/core/router_config.go index 1a33bb105b..38ea0ac910 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -26,7 +26,7 @@ import ( ) type subscriptionHooks struct { - startSubscription []func(ctx SubscriptionOnStartHookContext) (bool, error) + onStart []func(ctx SubscriptionOnStartHookContext) (bool, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 68fe78b6c5..3bbdc69abb 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -102,8 +102,8 @@ type SubscriptionOnStartHandler interface { SubscriptionOnStart(ctx SubscriptionOnStartHookContext) (bool, error) } -// NewPubSubOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a pubsub.OnSubscriptionStartFn -func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.OnSubscriptionStartFn { +// NewPubSubSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a pubsub.SubscriptionOnStartFn +func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.SubscriptionOnStartFn { if fn == nil { return nil } @@ -122,8 +122,8 @@ func NewPubSubOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext } } -// NewEngineOnSubscriptionStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.OnSubscriptionStartFn -func NewEngineOnSubscriptionStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.SubscriptionOnStartFn { +// NewEngineSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.SubscriptionOnStartFn +func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.SubscriptionOnStartFn { if fn == nil { return nil } diff --git a/router/pkg/pubsub/datasource/factory.go b/router/pkg/pubsub/datasource/factory.go index c7b371c244..90fb9cbd3b 100644 --- a/router/pkg/pubsub/datasource/factory.go +++ b/router/pkg/pubsub/datasource/factory.go @@ -11,14 +11,14 @@ import ( type PlannerConfig[PB ProviderBuilder[P, E], P any, E any] struct { ProviderBuilder PB Event E - OnSubscriptionStartFns []OnSubscriptionStartFn + SubscriptionOnStartFns []SubscriptionOnStartFn } -func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, onSubscriptionStartFns []OnSubscriptionStartFn) *PlannerConfig[PB, P, E] { +func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, subscriptionOnStartFns []SubscriptionOnStartFn) *PlannerConfig[PB, P, E] { return &PlannerConfig[PB, P, E]{ ProviderBuilder: providerBuilder, Event: event, - OnSubscriptionStartFns: onSubscriptionStartFns, + SubscriptionOnStartFns: subscriptionOnStartFns, } } diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go index dec868aaa7..6e6bed217b 100644 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ b/router/pkg/pubsub/datasource/hookeddatasource.go @@ -6,12 +6,12 @@ import ( ) type HookedSubscriptionDataSource struct { - OnSubscriptionStartFns []OnSubscriptionStartFn + SubscriptionOnStartFns []SubscriptionOnStartFn SubscriptionDataSource PubSubSubscriptionDataSource } func (h *HookedSubscriptionDataSource) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { - for _, fn := range h.OnSubscriptionStartFns { + for _, fn := range h.SubscriptionOnStartFns { close, err = fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) if err != nil || close { return diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index 4d2a49d3b2..c9e3b098b6 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -119,7 +119,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati hookedDataSource := &HookedSubscriptionDataSource{ SubscriptionDataSource: dataSource, - OnSubscriptionStartFns: p.config.OnSubscriptionStartFns, + SubscriptionOnStartFns: p.config.SubscriptionOnStartFns, } input, err := pubSubDataSource.ResolveDataSourceSubscriptionInput() diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index bd6f6cbc81..c72c816b0d 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -50,7 +50,7 @@ type StreamEvent interface { GetData() []byte } -type OnSubscriptionStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) +type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index b7a9da1d45..2b27e13363 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -119,6 +119,10 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) + if subConf == nil { + return fmt.Errorf("no subscription configuration found") + } + conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { return fmt.Errorf("invalid subscription configuration") diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 72779273bd..ffdced9827 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -116,6 +116,10 @@ func (s *SubscriptionSource) UniqueRequestID(ctx *resolve.Context, input []byte, func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) + if subConf == nil { + return fmt.Errorf("no subscription configuration found") + } + conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { return fmt.Errorf("invalid subscription configuration") diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index 1afc212662..f8901e3a50 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -50,7 +50,7 @@ func (e *ProviderNotDefinedError) Error() string { } type Hooks struct { - OnSubscriptionStarts []pubsub_datasource.OnSubscriptionStartFn + SubscriptionOnStart []pubsub_datasource.SubscriptionOnStartFn } // BuildProvidersAndDataSources is a generic function that builds providers and data sources for the given @@ -159,7 +159,7 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder // build data sources for each event for _, dsConf := range dsConfs { for i, event := range dsConf.events { - plannerConfig := pubsub_datasource.NewPlannerConfig(builder, event, hooks.OnSubscriptionStarts) + plannerConfig := pubsub_datasource.NewPlannerConfig(builder, event, hooks.SubscriptionOnStart) out, err := plan.NewDataSourceConfiguration( dsConf.dsConf.Configuration.Id+"-"+builder.TypeID()+"-"+strconv.Itoa(i), pubsub_datasource.NewPlannerFactory(ctx, plannerConfig), diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 51ad0963c3..cb97bafb9a 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -110,6 +110,10 @@ func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []b // Start starts the subscription func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { subConf := s.SubscriptionEventConfiguration(input) + if subConf == nil { + return fmt.Errorf("no subscription configuration found") + } + conf, ok := subConf.(*SubscriptionEventConfiguration) if !ok { return fmt.Errorf("invalid subscription configuration") From 45a71e07e14e3d167929058dd20a8288fff19807 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 24 Jul 2025 11:16:16 +0200 Subject: [PATCH 075/173] chore: add comments --- router/pkg/pubsub/datasource/subscription_event_updater.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 33e71da8ee..9332d10f7a 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -2,6 +2,9 @@ package datasource import "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +// SubscriptionEventUpdater is a wrapper around the SubscriptionUpdater interface +// that provides a way to send the event struct instead of the raw data +// It is used to give access to the event additional fields to the hooks. type SubscriptionEventUpdater interface { Update(event StreamEvent) Complete() From c58e79ae6630f2e67ccde105973b552c75e798d3 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 24 Jul 2025 11:33:00 +0200 Subject: [PATCH 076/173] chore: add headers on kafka publish --- router/pkg/pubsub/kafka/adapter.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index a100a3ebdf..17b805f72e 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -172,9 +172,19 @@ func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfigu var pErr error + headers := make([]kgo.RecordHeader, 0, len(event.Event.Headers)) + for key, value := range event.Event.Headers { + headers = append(headers, kgo.RecordHeader{ + Key: key, + Value: value, + }) + } + p.writeClient.Produce(ctx, &kgo.Record{ - Topic: event.Topic, - Value: event.Event.Data, + Key: event.Event.Key, + Topic: event.Topic, + Value: event.Event.Data, + Headers: headers, }, func(record *kgo.Record, err error) { defer wg.Done() if err != nil { From 7c4a5085c00de43a1fab721438f1364af6eb4b62 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 24 Jul 2025 18:50:01 +0200 Subject: [PATCH 077/173] chore: use new TryEmitSubscriptionUpdate, added Headers on nats --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/subscriptions_modules.go | 19 ++++++++++--------- router/go.mod | 2 +- router/go.sum | 4 ++-- router/pkg/pubsub/nats/adapter.go | 8 +++++++- router/pkg/pubsub/nats/engine_datasource.go | 4 ++-- 7 files changed, 25 insertions(+), 18 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 97796f26a8..011455bae6 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 193b125483..516e3c2e53 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c h1:3RAjYdKjtRegTrKkeqwQmrxWsoeyS27oYkjBeKygo/c= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 3bbdc69abb..18e1ace8ae 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -48,13 +48,14 @@ type SubscriptionOnStartHookContext interface { // the subscription event configuration (will return nil for engine subscription) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration // write an event to the stream of the current subscription - WriteEvent(event datasource.StreamEvent) + // returns true if the event was written to the stream, false if the event was dropped + WriteEvent(event datasource.StreamEvent) bool } type pubSubSubscriptionOnStartHookContext struct { requestContext RequestContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration - writeEventHook func(data []byte) + writeEventHook func(data []byte) bool } func (c *pubSubSubscriptionOnStartHookContext) RequestContext() RequestContext { @@ -65,8 +66,8 @@ func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() return c.subscriptionEventConfiguration } -func (c *pubSubSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - c.writeEventHook(event.GetData()) +func (c *pubSubSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { + return c.writeEventHook(event.GetData()) } // EngineEvent is the event used to write to the engine subscription @@ -80,15 +81,15 @@ func (e *EngineEvent) GetData() []byte { type engineSubscriptionOnStartHookContext struct { requestContext RequestContext - writeEventHook func(data []byte) + writeEventHook func(data []byte) bool } func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { return c.requestContext } -func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) { - c.writeEventHook(event.GetData()) +func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { + return c.writeEventHook(event.GetData()) } func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { @@ -113,7 +114,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext hookCtx := &pubSubSubscriptionOnStartHookContext{ requestContext: requestContext, subscriptionEventConfiguration: subConf, - writeEventHook: resolveCtx.EmitSubscriptionUpdate, + writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } close, err := fn(hookCtx) @@ -132,7 +133,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, - writeEventHook: resolveCtx.EmitSubscriptionUpdate, + writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } close, err := fn(hookCtx) diff --git a/router/go.mod b/router/go.mod index 20b5a63647..a643fd45e3 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index d17278fd8a..fca1956b6b 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c h1:3RAjYdKjtRegTrKkeqwQmrxWsoeyS27oYkjBeKygo/c= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250723153441-ec79b069058c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index bde0ec1bbe..065bd5769f 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -131,8 +131,14 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) + headers := make(map[string][]string) + for key, value := range msg.Headers() { + headers[key] = value + } + updater.Update(&Event{ - Data: msg.Data(), + Data: msg.Data(), + Headers: headers, }) // Acknowledge the message after it has been processed diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index ffdced9827..a00f30f4a4 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -16,8 +16,8 @@ import ( // Event represents an event from NATS type Event struct { - Data json.RawMessage `json:"data"` - Metadata map[string]string `json:"metadata"` + Data json.RawMessage `json:"data"` + Headers map[string][]string `json:"headers"` } func (e *Event) GetData() []byte { From 87faf101c2f4abb1f86a89cb7be608a373dfaf4a Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 24 Jul 2025 18:59:31 +0200 Subject: [PATCH 078/173] chore: headers were not managed when nats was using channels --- router/pkg/pubsub/nats/adapter.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 065bd5769f..4e332c056f 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -131,14 +131,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) - headers := make(map[string][]string) - for key, value := range msg.Headers() { - headers[key] = value - } - updater.Update(&Event{ Data: msg.Data(), - Headers: headers, + Headers: msg.Headers(), }) // Acknowledge the message after it has been processed @@ -177,7 +172,8 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent case msg := <-msgChan: log.Debug("subscription update", zap.String("message_subject", msg.Subject), zap.ByteString("data", msg.Data)) updater.Update(&Event{ - Data: msg.Data, + Data: msg.Data, + Headers: msg.Header, }) case <-p.ctx.Done(): // When the application context is done, we stop the subscriptions From dee05a80daa9c7276da15d38c6cba94ada3bff7b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 13:20:31 +0200 Subject: [PATCH 079/173] chore: initial working version of Batch and Stream hooks --- adr/cosmo-streams-v1.md | 2 - .../availability/subgraph/schema.resolvers.go | 14 +- .../mood/subgraph/schema.resolvers.go | 9 +- router-tests/events/kafka_events_test.go | 134 +++------ router-tests/events/utils.go | 68 +++++ router-tests/modules/stream-batch/module.go | 49 ++++ router-tests/modules/stream-publish/module.go | 49 ++++ router-tests/modules/stream_batch_test.go | 188 +++++++++++++ router-tests/modules/stream_publish_test.go | 106 +++++++ router/.mockery.yml | 6 - router/core/factoryresolver.go | 13 + router/core/router.go | 8 + router/core/router_config.go | 5 +- router/core/subscriptions_modules.go | 86 ++++++ router/pkg/pubsub/datasource/factory.go | 4 +- .../pkg/pubsub/datasource/hookedprovider.go | 90 ++++++ router/pkg/pubsub/datasource/mocks.go | 174 ++++++++++-- router/pkg/pubsub/datasource/planner.go | 4 +- router/pkg/pubsub/datasource/provider.go | 20 +- .../pkg/pubsub/datasource/pubsubprovider.go | 12 +- .../pubsub/datasource/pubsubprovider_test.go | 8 +- .../datasource/subscription_event_updater.go | 8 +- router/pkg/pubsub/kafka/adapter.go | 96 ++++--- router/pkg/pubsub/kafka/engine_datasource.go | 10 +- .../pubsub/kafka/engine_datasource_factory.go | 2 +- .../kafka/engine_datasource_factory_test.go | 5 +- .../pubsub/kafka/engine_datasource_test.go | 24 +- router/pkg/pubsub/kafka/mocks.go | 42 +-- router/pkg/pubsub/kafka/provider_builder.go | 20 +- router/pkg/pubsub/nats/adapter.go | 105 ++++--- router/pkg/pubsub/nats/engine_datasource.go | 8 +- .../pubsub/nats/engine_datasource_factory.go | 1 - router/pkg/pubsub/nats/mocks.go | 82 +++--- router/pkg/pubsub/nats/provider_builder.go | 17 +- router/pkg/pubsub/pubsub.go | 35 ++- router/pkg/pubsub/redis/adapter.go | 76 +++-- router/pkg/pubsub/redis/engine_datasource.go | 6 +- .../redis/engine_datasource_factory_test.go | 5 +- .../pubsub/redis/engine_datasource_test.go | 24 +- router/pkg/pubsub/redis/mocks.go | 261 ------------------ router/pkg/pubsub/redis/provider_builder.go | 11 +- 41 files changed, 1248 insertions(+), 639 deletions(-) create mode 100644 router-tests/events/utils.go create mode 100644 router-tests/modules/stream-batch/module.go create mode 100644 router-tests/modules/stream-publish/module.go create mode 100644 router-tests/modules/stream_batch_test.go create mode 100644 router-tests/modules/stream_publish_test.go create mode 100644 router/pkg/pubsub/datasource/hookedprovider.go delete mode 100644 router/pkg/pubsub/redis/mocks.go diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index d5929b8538..42a78bb64c 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -69,8 +69,6 @@ type PublishEventConfiguration interface { type SubscriptionOnStartHookContext interface { // the request context RequestContext() RequestContext - // the stream context - StreamContext() StreamContext // the subscription event configuration SubscriptionEventConfiguration() SubscriptionEventConfiguration // write an event to the stream of the current subscription diff --git a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go index 6dd313493e..c768f446d7 100644 --- a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go @@ -10,24 +10,30 @@ import ( "github.com/wundergraph/cosmo/demo/pkg/subgraphs/availability/subgraph/generated" "github.com/wundergraph/cosmo/demo/pkg/subgraphs/availability/subgraph/model" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" ) // UpdateAvailability is the resolver for the updateAvailability field. func (r *mutationResolver) UpdateAvailability(ctx context.Context, employeeID int, isAvailable bool) (*model.Employee, error) { storage.Set(employeeID, isAvailable) - err := r.NatsPubSubByProviderID["default"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ + conf := &nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdated.%d", employeeID)), Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, - }) + } + evt := &nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))} + err := r.NatsPubSubByProviderID["default"].Publish(ctx, conf, []datasource.StreamEvent{evt}) if err != nil { return nil, err } - err = r.NatsPubSubByProviderID["my-nats"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ + + conf2 := &nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdatedMyNats.%d", employeeID)), Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, - }) + } + evt2 := &nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))} + err = r.NatsPubSubByProviderID["my-nats"].Publish(ctx, conf2, []datasource.StreamEvent{evt2}) if err != nil { return nil, err diff --git a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go index 82a0a7e9f2..6417798b16 100644 --- a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go @@ -10,6 +10,7 @@ import ( "github.com/wundergraph/cosmo/demo/pkg/subgraphs/mood/subgraph/generated" "github.com/wundergraph/cosmo/demo/pkg/subgraphs/mood/subgraph/model" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/nats" ) @@ -19,10 +20,10 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood myNatsTopic := r.GetPubSubName(fmt.Sprintf("employeeUpdated.%d", employeeID)) payload := fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID) if r.NatsPubSubByProviderID["default"] != nil { - err := r.NatsPubSubByProviderID["default"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ + err := r.NatsPubSubByProviderID["default"].Publish(ctx, &nats.PublishAndRequestEventConfiguration{ Subject: myNatsTopic, Event: nats.Event{Data: []byte(payload)}, - }) + }, []datasource.StreamEvent{&nats.Event{Data: []byte(payload)}}) if err != nil { return nil, err } @@ -32,10 +33,10 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood defaultTopic := r.GetPubSubName(fmt.Sprintf("employeeUpdatedMyNats.%d", employeeID)) if r.NatsPubSubByProviderID["my-nats"] != nil { - err := r.NatsPubSubByProviderID["my-nats"].Publish(ctx, nats.PublishAndRequestEventConfiguration{ + err := r.NatsPubSubByProviderID["my-nats"].Publish(ctx, &nats.PublishAndRequestEventConfiguration{ Subject: defaultTopic, Event: nats.Event{Data: []byte(payload)}, - }) + }, []datasource.StreamEvent{&nats.Event{Data: []byte(payload)}}) if err != nil { return nil, err } diff --git a/router-tests/events/kafka_events_test.go b/router-tests/events/kafka_events_test.go index 05f4250003..ffd08736c4 100644 --- a/router-tests/events/kafka_events_test.go +++ b/router-tests/events/kafka_events_test.go @@ -3,7 +3,6 @@ package events_test import ( "bufio" "bytes" - "context" "encoding/json" "fmt" "net/http" @@ -16,7 +15,7 @@ import ( "github.com/hasura/go-graphql-client" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" + "github.com/wundergraph/cosmo/router-tests/events" "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/pkg/config" ) @@ -74,7 +73,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -107,7 +106,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) @@ -130,7 +129,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -164,23 +163,23 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], ``) // Empty message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], ``) // Empty message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.ErrorContains(t, args.errValue, "Invalid message received") }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) // Missing entity = Resolver error + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) // Missing entity = Resolver error testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.ErrorContains(t, args.errValue, "Cannot return null for non-nullable field 'Subscription.employeeUpdatedMyKafka.id'.") }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) @@ -204,7 +203,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -248,7 +247,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(2, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) @@ -277,7 +276,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -321,7 +320,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(2, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) @@ -333,7 +332,7 @@ func TestKafkaEvents(t *testing.T) { require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) }) - produceKafkaMessage(t, xEnv, topics[1], `{"__typename":"Employee","id": 2,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[1], `{"__typename":"Employee","id": 2,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) @@ -366,7 +365,7 @@ func TestKafkaEvents(t *testing.T) { engineExecutionConfiguration.WebSocketClientReadTimeout = time.Millisecond * 100 }, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -399,7 +398,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) @@ -431,7 +430,7 @@ func TestKafkaEvents(t *testing.T) { core.WithMultipartHeartbeatInterval(multipartHeartbeatInterval), }, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -447,10 +446,10 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) assertKafkaMultipartValueEventually(t, reader, "{\"payload\":{\"data\":{\"employeeUpdatedMyKafka\":{\"id\":1,\"details\":{\"forename\":\"Jens\",\"surname\":\"Neuse\"}}}}}") - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) assertKafkaMultipartValueEventually(t, reader, "{\"payload\":{\"data\":{\"employeeUpdatedMyKafka\":{\"id\":1,\"details\":{\"forename\":\"Jens\",\"surname\":\"Neuse\"}}}}}") }) }) @@ -497,7 +496,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -530,7 +529,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, responseCh, func(t *testing.T, response struct { response *http.Response @@ -562,7 +561,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -595,7 +594,7 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) testenv.AwaitChannelWithT(t, KafkaWaitTimeout, responseCh, func(t *testing.T, resp struct { response *http.Response @@ -672,7 +671,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -713,7 +712,7 @@ func TestKafkaEvents(t *testing.T) { // Events 1, 2, 11, and 12 should be included for i := uint32(1); i < 13; i++ { - produceKafkaMessage(t, xEnv, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) if i == 1 || i == 2 || i == 11 || i == 12 { conn.SetReadDeadline(time.Now().Add(KafkaWaitTimeout)) gErr := conn.ReadJSON(&msg) @@ -739,7 +738,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -780,7 +779,7 @@ func TestKafkaEvents(t *testing.T) { // Events 1, 2, 11, and 12 should be included for i := uint32(1); i < 13; i++ { - produceKafkaMessage(t, xEnv, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) if i == 1 || i == 2 || i == 11 || i == 12 { conn.SetReadDeadline(time.Now().Add(KafkaWaitTimeout)) gErr := conn.ReadJSON(&msg) @@ -806,7 +805,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -835,10 +834,10 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) // The message should be ignored because "1" does not equal 1 - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id":1}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id":1}`) // This message should be delivered because it matches the filter - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id":12}`) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id":12}`) conn.SetReadDeadline(time.Now().Add(KafkaWaitTimeout)) readErr := conn.ReadJSON(&msg) require.NoError(t, readErr) @@ -861,7 +860,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -894,23 +893,23 @@ func TestKafkaEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, KafkaWaitTimeout) - produceKafkaMessage(t, xEnv, topics[0], `{asas`) // Invalid message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{asas`) // Invalid message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.ErrorContains(t, args.errValue, "Invalid message received") }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id":1}`) // Correct message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id":1}`) // Correct message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) // Missing entity = Resolver error + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) // Missing entity = Resolver error testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.ErrorContains(t, args.errValue, "Cannot return null for non-nullable field 'Subscription.employeeUpdatedMyKafka.id'.") }) - produceKafkaMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message testenv.AwaitChannelWithT(t, KafkaWaitTimeout, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) @@ -932,7 +931,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) // Send a mutation to trigger the first subscription resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ @@ -940,7 +939,7 @@ func TestKafkaEvents(t *testing.T) { }) require.JSONEq(t, `{"data":{"updateEmployeeMyKafka":{"success":true}}}`, resOne.Body) - records, err := readKafkaMessages(xEnv, topics[0], 1) + records, err := events.ReadKafkaMessages(xEnv, KafkaWaitTimeout, topics[0], 1) require.NoError(t, err) require.Equal(t, 1, len(records)) require.Equal(t, `{"employeeID":3,"update":{"name":"name test"}}`, string(records[0].Value)) @@ -980,7 +979,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - ensureTopicExists(t, xEnv, topics...) + events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -1024,7 +1023,7 @@ func TestKafkaEvents(t *testing.T) { // Events 1, 3, 4, 7, and 11 should be included for i := int(MsgCount); i > 0; i-- { - produceKafkaMessage(t, xEnv, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) + events.ProduceKafkaMessage(t, xEnv, KafkaWaitTimeout, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) if i == 1 || i == 3 || i == 4 || i == 7 || i == 11 { conn.SetReadDeadline(time.Now().Add(KafkaWaitTimeout)) jsonErr := conn.ReadJSON(&msg) @@ -1041,60 +1040,3 @@ func TestKafkaEvents(t *testing.T) { }) }) } - -func ensureTopicExists(t *testing.T, xEnv *testenv.Environment, topics ...string) { - // Delete topic for idempotency - deleteCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - prefixedTopics := make([]string, len(topics)) - for _, topic := range topics { - prefixedTopics = append(prefixedTopics, xEnv.GetPubSubName(topic)) - } - - _, err := xEnv.KafkaAdminClient.DeleteTopics(deleteCtx, prefixedTopics...) - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - _, err = xEnv.KafkaAdminClient.CreateTopics(ctx, 1, 1, nil, prefixedTopics...) - require.NoError(t, err) -} - -func produceKafkaMessage(t *testing.T, xEnv *testenv.Environment, topicName string, message string) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pErrCh := make(chan error) - - xEnv.KafkaClient.Produce(ctx, &kgo.Record{ - Topic: xEnv.GetPubSubName(topicName), - Value: []byte(message), - }, func(record *kgo.Record, err error) { - pErrCh <- err - }) - - testenv.AwaitChannelWithT(t, KafkaWaitTimeout, pErrCh, func(t *testing.T, pErr error) { - require.NoError(t, pErr) - }) - - fErr := xEnv.KafkaClient.Flush(ctx) - require.NoError(t, fErr) -} - -func readKafkaMessages(xEnv *testenv.Environment, topicName string, msgs int) ([]*kgo.Record, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - client, err := kgo.NewClient( - kgo.SeedBrokers(xEnv.GetKafkaSeeds()...), - kgo.ConsumeTopics(xEnv.GetPubSubName(topicName)), - ) - if err != nil { - return nil, err - } - - fetchs := client.PollRecords(ctx, msgs) - - return fetchs.Records(), nil -} diff --git a/router-tests/events/utils.go b/router-tests/events/utils.go new file mode 100644 index 0000000000..071bc92128 --- /dev/null +++ b/router-tests/events/utils.go @@ -0,0 +1,68 @@ +package events + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/wundergraph/cosmo/router-tests/testenv" +) + +func EnsureTopicExists(t *testing.T, xEnv *testenv.Environment, timeout time.Duration, topics ...string) { + // Delete topic for idempotency + deleteCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + prefixedTopics := make([]string, len(topics)) + for _, topic := range topics { + prefixedTopics = append(prefixedTopics, xEnv.GetPubSubName(topic)) + } + + _, err := xEnv.KafkaAdminClient.DeleteTopics(deleteCtx, prefixedTopics...) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + _, err = xEnv.KafkaAdminClient.CreateTopics(ctx, 1, 1, nil, prefixedTopics...) + require.NoError(t, err) +} + +func ProduceKafkaMessage(t *testing.T, xEnv *testenv.Environment, timeout time.Duration, topicName string, message string) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + pErrCh := make(chan error) + + xEnv.KafkaClient.Produce(ctx, &kgo.Record{ + Topic: xEnv.GetPubSubName(topicName), + Value: []byte(message), + }, func(record *kgo.Record, err error) { + pErrCh <- err + }) + + testenv.AwaitChannelWithT(t, timeout, pErrCh, func(t *testing.T, pErr error) { + require.NoError(t, pErr) + }) + + fErr := xEnv.KafkaClient.Flush(ctx) + require.NoError(t, fErr) +} + +func ReadKafkaMessages(xEnv *testenv.Environment, timeout time.Duration, topicName string, msgs int) ([]*kgo.Record, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + client, err := kgo.NewClient( + kgo.SeedBrokers(xEnv.GetKafkaSeeds()...), + kgo.ConsumeTopics(xEnv.GetPubSubName(topicName)), + ) + if err != nil { + return nil, err + } + + fetchs := client.PollRecords(ctx, msgs) + + return fetchs.Records(), nil +} diff --git a/router-tests/modules/stream-batch/module.go b/router-tests/modules/stream-batch/module.go new file mode 100644 index 0000000000..cd62ec09c0 --- /dev/null +++ b/router-tests/modules/stream-batch/module.go @@ -0,0 +1,49 @@ +package batch + +import ( + "go.uber.org/zap" + + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" +) + +const myModuleID = "streamBatchModule" + +type StreamBatchModule struct { + Logger *zap.Logger + Callback func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) +} + +func (m *StreamBatchModule) Provision(ctx *core.ModuleContext) error { + // Assign the logger to the module for non-request related logging + m.Logger = ctx.Logger + + return nil +} + +func (m *StreamBatchModule) OnStreamEvents(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + m.Logger.Info("Stream Hook has been run") + + if m.Callback != nil { + return m.Callback(ctx, events) + } + + return events, nil +} + +func (m *StreamBatchModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + // This is the ID of your module, it must be unique + ID: myModuleID, + // The priority of your module, lower numbers are executed first + Priority: 1, + New: func() core.Module { + return &StreamBatchModule{} + }, + } +} + +// Interface guard +var ( + _ core.StreamBatchEventHook = (*StreamBatchModule)(nil) +) diff --git a/router-tests/modules/stream-publish/module.go b/router-tests/modules/stream-publish/module.go new file mode 100644 index 0000000000..827868903f --- /dev/null +++ b/router-tests/modules/stream-publish/module.go @@ -0,0 +1,49 @@ +package publish + +import ( + "go.uber.org/zap" + + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" +) + +const myModuleID = "publishModule" + +type PublishModule struct { + Logger *zap.Logger + Callback func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) +} + +func (m *PublishModule) Provision(ctx *core.ModuleContext) error { + // Assign the logger to the module for non-request related logging + m.Logger = ctx.Logger + + return nil +} + +func (m *PublishModule) OnPublishEvents(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + m.Logger.Info("Publish Hook has been run") + + if m.Callback != nil { + return m.Callback(ctx, events) + } + + return events, nil +} + +func (m *PublishModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + // This is the ID of your module, it must be unique + ID: myModuleID, + // The priority of your module, lower numbers are executed first + Priority: 1, + New: func() core.Module { + return &PublishModule{} + }, + } +} + +// Interface guard +var ( + _ core.StreamPublishEventHook = (*PublishModule)(nil) +) diff --git a/router-tests/modules/stream_batch_test.go b/router-tests/modules/stream_batch_test.go new file mode 100644 index 0000000000..99ac1a1711 --- /dev/null +++ b/router-tests/modules/stream_batch_test.go @@ -0,0 +1,188 @@ +package module_test + +import ( + "testing" + "time" + + "go.uber.org/zap/zapcore" + + "github.com/hasura/go-graphql-client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router-tests/events" + stream_batch "github.com/wundergraph/cosmo/router-tests/modules/stream-batch" + "github.com/wundergraph/cosmo/router-tests/testenv" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/config" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" +) + +func TestBatchHook(t *testing.T) { + t.Parallel() + + const Timeout = time.Second * 10 + + type kafkaSubscriptionArgs struct { + dataValue []byte + errValue error + } + + t.Run("Test Batch hook is called", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "batchModule": stream_batch.StreamBatchModule{}, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_batch.StreamBatchModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.EnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("Stream Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) + + t.Run("Test Batch hook could change events", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "streamBatchModule": stream_batch.StreamBatchModule{ + Callback: func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + evt.Data = []byte(`{"__typename":"Employee","id": 3,"update":{"name":"foo"}}`) + } + + return events, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_batch.StreamBatchModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.EnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":3,"details":{"forename":"Stefan","surname":"Avram"}}}`, string(args.dataValue)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("Stream Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) +} diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go new file mode 100644 index 0000000000..faa6d9856b --- /dev/null +++ b/router-tests/modules/stream_publish_test.go @@ -0,0 +1,106 @@ +package module_test + +import ( + "testing" + "time" + + "go.uber.org/zap/zapcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router-tests/events" + stream_publish "github.com/wundergraph/cosmo/router-tests/modules/stream-publish" + "github.com/wundergraph/cosmo/router-tests/testenv" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/config" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" +) + +func TestPublishHook(t *testing.T) { + t.Parallel() + + t.Run("Test Publish hook is called", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{}, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, + }) + require.JSONEq(t, `{"data":{"updateEmployeeMyKafka":{"success":false}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) + + t.Run("Test Publish kafka hook allows to set headers", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + evt.Headers["x-test"] = []byte("test") + } + + return events, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + events.EnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, + }) + require.JSONEq(t, `{"data":{"updateEmployeeMyKafka":{"success":true}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) + require.NoError(t, err) + require.Len(t, records, 1) + header := records[0].Headers[0] + require.Equal(t, "x-test", header.Key) + require.Equal(t, []byte("test"), header.Value) + }) + }) +} diff --git a/router/.mockery.yml b/router/.mockery.yml index 8ea750cc0e..b84fa2f4fa 100644 --- a/router/.mockery.yml +++ b/router/.mockery.yml @@ -21,12 +21,6 @@ packages: github.com/wundergraph/cosmo/router/pkg/pubsub/nats: interfaces: Adapter: - github.com/wundergraph/cosmo/router/pkg/pubsub/kafka: - interfaces: - Adapter: - github.com/wundergraph/cosmo/router/pkg/pubsub/redis: - interfaces: - Adapter: github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve: config: dir: 'pkg/pubsub/datasource' diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 6827834db4..59a85b6047 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -478,6 +478,17 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod for i, fn := range l.subscriptionHooks.onStart { subscriptionOnStartFns[i] = NewPubSubSubscriptionOnStartHook(fn) } + + onPublishEventsFns := make([]pubsub_datasource.OnPublishEventsFn, len(l.subscriptionHooks.onPublishEvents)) + for i, fn := range l.subscriptionHooks.onPublishEvents { + onPublishEventsFns[i] = NewPubSubOnPublishEventsHook(fn) + } + + onStreamEventsFns := make([]pubsub_datasource.OnStreamEventsFn, len(l.subscriptionHooks.onStreamEvents)) + for i, fn := range l.subscriptionHooks.onStreamEvents { + onStreamEventsFns[i] = NewPubSubOnStreamEventsHook(fn) + } + factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, routerEngineConfig.Events, @@ -487,6 +498,8 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod l.resolver.InstanceData().ListenAddress, pubsub.Hooks{ SubscriptionOnStart: subscriptionOnStartFns, + OnStreamEvents: onStreamEventsFns, + OnPublishEvents: onPublishEventsFns, }, ) if err != nil { diff --git a/router/core/router.go b/router/core/router.go index 1d1414470a..39684a730a 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -646,6 +646,14 @@ func (r *Router) initModules(ctx context.Context) error { r.subscriptionHooks.onStart = append(r.subscriptionHooks.onStart, handler.SubscriptionOnStart) } + if handler, ok := moduleInstance.(StreamPublishEventHook); ok { + r.subscriptionHooks.onPublishEvents = append(r.subscriptionHooks.onPublishEvents, handler.OnPublishEvents) + } + + if handler, ok := moduleInstance.(StreamBatchEventHook); ok { + r.subscriptionHooks.onStreamEvents = append(r.subscriptionHooks.onStreamEvents, handler.OnStreamEvents) + } + r.modules = append(r.modules, moduleInstance) r.logger.Info("Module registered", diff --git a/router/core/router_config.go b/router/core/router_config.go index 38ea0ac910..f4b0195f25 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -17,6 +17,7 @@ import ( "github.com/wundergraph/cosmo/router/pkg/health" "github.com/wundergraph/cosmo/router/pkg/mcpserver" rmetric "github.com/wundergraph/cosmo/router/pkg/metric" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" rtrace "github.com/wundergraph/cosmo/router/pkg/trace" "go.opentelemetry.io/otel/propagation" sdkmetric "go.opentelemetry.io/otel/sdk/metric" @@ -26,7 +27,9 @@ import ( ) type subscriptionHooks struct { - onStart []func(ctx SubscriptionOnStartHookContext) (bool, error) + onStart []func(ctx SubscriptionOnStartHookContext) (bool, error) + onPublishEvents []func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + onStreamEvents []func(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 18e1ace8ae..79a2392c29 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -1,6 +1,8 @@ package core import ( + "context" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" @@ -52,6 +54,19 @@ type SubscriptionOnStartHookContext interface { WriteEvent(event datasource.StreamEvent) bool } +type pubSubPublishEventHookContext struct { + requestContext RequestContext + publishEventConfiguration datasource.PublishEventConfiguration +} + +func (c *pubSubPublishEventHookContext) RequestContext() RequestContext { + return c.requestContext +} + +func (c *pubSubPublishEventHookContext) PublishEventConfiguration() datasource.PublishEventConfiguration { + return c.publishEventConfiguration +} + type pubSubSubscriptionOnStartHookContext struct { requestContext RequestContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration @@ -141,3 +156,74 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext return close, err } } + +type StreamBatchEventHookContext interface { + // the request context + RequestContext() RequestContext + // the subscription event configuration + SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration +} + +type StreamBatchEventHook interface { + // OnStreamEvents is called each time a batch of events is received from the provider + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + OnStreamEvents(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) +} + +type StreamPublishEventHookContext interface { + // the request context + RequestContext() RequestContext + // the publish event configuration + PublishEventConfiguration() datasource.PublishEventConfiguration +} + +type StreamPublishEventHook interface { + // OnPublishEvents is called each time a batch of events is going to be sent to the provider + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + OnPublishEvents(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) +} + +func NewPubSubOnPublishEventsHook(fn func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnPublishEventsFn { + if fn == nil { + return nil + } + + return func(ctx context.Context, pubConf datasource.PublishEventConfiguration, evts []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + requestContext := getRequestContext(ctx) + hookCtx := &pubSubPublishEventHookContext{ + requestContext: requestContext, + publishEventConfiguration: pubConf, + } + + return fn(hookCtx, evts) + } +} + +type pubSubStreamBatchEventHookContext struct { + requestContext RequestContext + subscriptionEventConfiguration datasource.SubscriptionEventConfiguration +} + +func (c *pubSubStreamBatchEventHookContext) RequestContext() RequestContext { + return c.requestContext +} + +func (c *pubSubStreamBatchEventHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { + return c.subscriptionEventConfiguration +} + +func NewPubSubOnStreamEventsHook(fn func(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnStreamEventsFn { + if fn == nil { + return nil + } + + return func(ctx context.Context, subConf datasource.SubscriptionEventConfiguration, evts []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + requestContext := getRequestContext(ctx) + hookCtx := &pubSubStreamBatchEventHookContext{ + requestContext: requestContext, + subscriptionEventConfiguration: subConf, + } + + return fn(hookCtx, evts) + } +} diff --git a/router/pkg/pubsub/datasource/factory.go b/router/pkg/pubsub/datasource/factory.go index 90fb9cbd3b..2431bb21a5 100644 --- a/router/pkg/pubsub/datasource/factory.go +++ b/router/pkg/pubsub/datasource/factory.go @@ -9,13 +9,15 @@ import ( ) type PlannerConfig[PB ProviderBuilder[P, E], P any, E any] struct { + Providers map[string]Provider ProviderBuilder PB Event E SubscriptionOnStartFns []SubscriptionOnStartFn } -func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, subscriptionOnStartFns []SubscriptionOnStartFn) *PlannerConfig[PB, P, E] { +func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, providers map[string]Provider, subscriptionOnStartFns []SubscriptionOnStartFn) *PlannerConfig[PB, P, E] { return &PlannerConfig[PB, P, E]{ + Providers: providers, ProviderBuilder: providerBuilder, Event: event, SubscriptionOnStartFns: subscriptionOnStartFns, diff --git a/router/pkg/pubsub/datasource/hookedprovider.go b/router/pkg/pubsub/datasource/hookedprovider.go new file mode 100644 index 0000000000..2518dcf1e1 --- /dev/null +++ b/router/pkg/pubsub/datasource/hookedprovider.go @@ -0,0 +1,90 @@ +package datasource + +import ( + "context" + + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +type hookedUpdater struct { + ctx context.Context + updater SubscriptionEventUpdater + subscriptionEventConfiguration SubscriptionEventConfiguration + OnStreamEventsFns []OnStreamEventsFn +} + +func (h *hookedUpdater) Update(events []StreamEvent) { + var newEvents []StreamEvent + var err error + for _, fn := range h.OnStreamEventsFns { + newEvents, err = fn(h.ctx, h.subscriptionEventConfiguration, events) + if err != nil { + // TODO: do something with the error + return + } + } + + h.updater.Update(newEvents) +} + +func (h *hookedUpdater) Complete() { + h.updater.Complete() +} + +func (h *hookedUpdater) Close(kind resolve.SubscriptionCloseKind) { + h.updater.Close(kind) +} + +func NewHookedProvider(provider Provider, onStreamEventsFns []OnStreamEventsFn, onPublishEventsFns []OnPublishEventsFn) Provider { + return &HookedProvider{ + OnStreamEventsFns: onStreamEventsFns, + OnPublishEventsFns: onPublishEventsFns, + Provider: provider, + } +} + +type HookedProvider struct { + Provider + OnPublishEventsFns []OnPublishEventsFn + OnStreamEventsFns []OnStreamEventsFn +} + +func (h *HookedProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + hookedUpdater := &hookedUpdater{ + ctx: ctx, + updater: updater, + subscriptionEventConfiguration: cfg, + OnStreamEventsFns: h.OnStreamEventsFns, + } + + return h.Provider.Subscribe(ctx, cfg, hookedUpdater) +} + +func (h *HookedProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { + var newEvents []StreamEvent + var err error + for _, fn := range h.OnPublishEventsFns { + newEvents, err = fn(ctx, cfg, events) + if err != nil { + return err + } + } + + return h.Provider.Publish(ctx, cfg, newEvents) +} + +func (h *HookedProvider) ID() string { + return h.Provider.ID() +} + +func (h *HookedProvider) TypeID() string { + return h.Provider.TypeID() +} + +func (h *HookedProvider) Startup(ctx context.Context) error { + return h.Provider.Startup(ctx) +} + +func (h *HookedProvider) Shutdown(ctx context.Context) error { + return h.Provider.Shutdown(ctx) +} diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index 1968b4bba1..c6b1153429 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -556,6 +556,69 @@ func (_c *MockProvider_ID_Call) RunAndReturn(run func() string) *MockProvider_ID return _c } +// Publish provides a mock function for the type MockProvider +func (_mock *MockProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { + ret := _mock.Called(ctx, cfg, events) + + if len(ret) == 0 { + panic("no return value specified for Publish") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, PublishEventConfiguration, []StreamEvent) error); ok { + r0 = returnFunc(ctx, cfg, events) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockProvider_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' +type MockProvider_Publish_Call struct { + *mock.Call +} + +// Publish is a helper method to define mock.On call +// - ctx context.Context +// - cfg PublishEventConfiguration +// - events []StreamEvent +func (_e *MockProvider_Expecter) Publish(ctx interface{}, cfg interface{}, events interface{}) *MockProvider_Publish_Call { + return &MockProvider_Publish_Call{Call: _e.mock.On("Publish", ctx, cfg, events)} +} + +func (_c *MockProvider_Publish_Call) Run(run func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent)) *MockProvider_Publish_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 PublishEventConfiguration + if args[1] != nil { + arg1 = args[1].(PublishEventConfiguration) + } + var arg2 []StreamEvent + if args[2] != nil { + arg2 = args[2].([]StreamEvent) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockProvider_Publish_Call) Return(err error) *MockProvider_Publish_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockProvider_Publish_Call) RunAndReturn(run func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error) *MockProvider_Publish_Call { + _c.Call.Return(run) + return _c +} + // Shutdown provides a mock function for the type MockProvider func (_mock *MockProvider) Shutdown(ctx context.Context) error { ret := _mock.Called(ctx) @@ -658,6 +721,69 @@ func (_c *MockProvider_Startup_Call) RunAndReturn(run func(ctx context.Context) return _c } +// Subscribe provides a mock function for the type MockProvider +func (_mock *MockProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + ret := _mock.Called(ctx, cfg, updater) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, SubscriptionEventUpdater) error); ok { + r0 = returnFunc(ctx, cfg, updater) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockProvider_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe' +type MockProvider_Subscribe_Call struct { + *mock.Call +} + +// Subscribe is a helper method to define mock.On call +// - ctx context.Context +// - cfg SubscriptionEventConfiguration +// - updater SubscriptionEventUpdater +func (_e *MockProvider_Expecter) Subscribe(ctx interface{}, cfg interface{}, updater interface{}) *MockProvider_Subscribe_Call { + return &MockProvider_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, cfg, updater)} +} + +func (_c *MockProvider_Subscribe_Call) Run(run func(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater)) *MockProvider_Subscribe_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 SubscriptionEventConfiguration + if args[1] != nil { + arg1 = args[1].(SubscriptionEventConfiguration) + } + var arg2 SubscriptionEventUpdater + if args[2] != nil { + arg2 = args[2].(SubscriptionEventUpdater) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockProvider_Subscribe_Call) Return(err error) *MockProvider_Subscribe_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockProvider_Subscribe_Call) RunAndReturn(run func(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error) *MockProvider_Subscribe_Call { + _c.Call.Return(run) + return _c +} + // TypeID provides a mock function for the type MockProvider func (_mock *MockProvider) TypeID() string { ret := _mock.Called() @@ -730,8 +856,8 @@ func (_m *MockProviderBuilder[P, E]) EXPECT() *MockProviderBuilder_Expecter[P, E } // BuildEngineDataSourceFactory provides a mock function for the type MockProviderBuilder -func (_mock *MockProviderBuilder[P, E]) BuildEngineDataSourceFactory(data E) (EngineDataSourceFactory, error) { - ret := _mock.Called(data) +func (_mock *MockProviderBuilder[P, E]) BuildEngineDataSourceFactory(data E, providers map[string]Provider) (EngineDataSourceFactory, error) { + ret := _mock.Called(data, providers) if len(ret) == 0 { panic("no return value specified for BuildEngineDataSourceFactory") @@ -739,18 +865,18 @@ func (_mock *MockProviderBuilder[P, E]) BuildEngineDataSourceFactory(data E) (En var r0 EngineDataSourceFactory var r1 error - if returnFunc, ok := ret.Get(0).(func(E) (EngineDataSourceFactory, error)); ok { - return returnFunc(data) + if returnFunc, ok := ret.Get(0).(func(E, map[string]Provider) (EngineDataSourceFactory, error)); ok { + return returnFunc(data, providers) } - if returnFunc, ok := ret.Get(0).(func(E) EngineDataSourceFactory); ok { - r0 = returnFunc(data) + if returnFunc, ok := ret.Get(0).(func(E, map[string]Provider) EngineDataSourceFactory); ok { + r0 = returnFunc(data, providers) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(EngineDataSourceFactory) } } - if returnFunc, ok := ret.Get(1).(func(E) error); ok { - r1 = returnFunc(data) + if returnFunc, ok := ret.Get(1).(func(E, map[string]Provider) error); ok { + r1 = returnFunc(data, providers) } else { r1 = ret.Error(1) } @@ -764,18 +890,24 @@ type MockProviderBuilder_BuildEngineDataSourceFactory_Call[P any, E any] struct // BuildEngineDataSourceFactory is a helper method to define mock.On call // - data E -func (_e *MockProviderBuilder_Expecter[P, E]) BuildEngineDataSourceFactory(data interface{}) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { - return &MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]{Call: _e.mock.On("BuildEngineDataSourceFactory", data)} +// - providers map[string]Provider +func (_e *MockProviderBuilder_Expecter[P, E]) BuildEngineDataSourceFactory(data interface{}, providers interface{}) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { + return &MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]{Call: _e.mock.On("BuildEngineDataSourceFactory", data, providers)} } -func (_c *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]) Run(run func(data E)) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { +func (_c *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]) Run(run func(data E, providers map[string]Provider)) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { _c.Call.Run(func(args mock.Arguments) { var arg0 E if args[0] != nil { arg0 = args[0].(E) } + var arg1 map[string]Provider + if args[1] != nil { + arg1 = args[1].(map[string]Provider) + } run( arg0, + arg1, ) }) return _c @@ -786,7 +918,7 @@ func (_c *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]) Return(en return _c } -func (_c *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]) RunAndReturn(run func(data E) (EngineDataSourceFactory, error)) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { +func (_c *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E]) RunAndReturn(run func(data E, providers map[string]Provider) (EngineDataSourceFactory, error)) *MockProviderBuilder_BuildEngineDataSourceFactory_Call[P, E] { _c.Call.Return(run) return _c } @@ -998,8 +1130,8 @@ func (_c *MockSubscriptionEventUpdater_Complete_Call) RunAndReturn(run func()) * } // Update provides a mock function for the type MockSubscriptionEventUpdater -func (_mock *MockSubscriptionEventUpdater) Update(event StreamEvent) { - _mock.Called(event) +func (_mock *MockSubscriptionEventUpdater) Update(events []StreamEvent) { + _mock.Called(events) return } @@ -1009,16 +1141,16 @@ type MockSubscriptionEventUpdater_Update_Call struct { } // Update is a helper method to define mock.On call -// - event StreamEvent -func (_e *MockSubscriptionEventUpdater_Expecter) Update(event interface{}) *MockSubscriptionEventUpdater_Update_Call { - return &MockSubscriptionEventUpdater_Update_Call{Call: _e.mock.On("Update", event)} +// - events []StreamEvent +func (_e *MockSubscriptionEventUpdater_Expecter) Update(events interface{}) *MockSubscriptionEventUpdater_Update_Call { + return &MockSubscriptionEventUpdater_Update_Call{Call: _e.mock.On("Update", events)} } -func (_c *MockSubscriptionEventUpdater_Update_Call) Run(run func(event StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { +func (_c *MockSubscriptionEventUpdater_Update_Call) Run(run func(events []StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 StreamEvent + var arg0 []StreamEvent if args[0] != nil { - arg0 = args[0].(StreamEvent) + arg0 = args[0].([]StreamEvent) } run( arg0, @@ -1032,7 +1164,7 @@ func (_c *MockSubscriptionEventUpdater_Update_Call) Return() *MockSubscriptionEv return _c } -func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(event StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { +func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(events []StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { _c.Run(run) return _c } diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index c9e3b098b6..471d19c789 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -55,7 +55,7 @@ func (p *Planner[PB, P, E]) ConfigureFetch() resolve.FetchConfiguration { return resolve.FetchConfiguration{} } - pubSubDataSource, err := p.config.ProviderBuilder.BuildEngineDataSourceFactory(p.config.Event) + pubSubDataSource, err := p.config.ProviderBuilder.BuildEngineDataSourceFactory(p.config.Event, p.config.Providers) if err != nil { p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to build data source: %w", err)) return resolve.FetchConfiguration{} @@ -100,7 +100,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati return plan.SubscriptionConfiguration{} } - pubSubDataSource, err := p.config.ProviderBuilder.BuildEngineDataSourceFactory(p.config.Event) + pubSubDataSource, err := p.config.ProviderBuilder.BuildEngineDataSourceFactory(p.config.Event, p.config.Providers) if err != nil { p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to get resolve data source subscription: %w", err)) return plan.SubscriptionConfiguration{} diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index c72c816b0d..e103be53b1 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -8,6 +8,8 @@ import ( type ArgumentTemplateCallback func(tpl string) (string, error) +// ProviderLifecycle is the interface that the provider must implement +// to allow the router to start and stop the provider type ProviderLifecycle interface { // Startup is the method called when the provider is started Startup(ctx context.Context) error @@ -15,9 +17,17 @@ type ProviderLifecycle interface { Shutdown(ctx context.Context) error } +// ProviderBase is the interface that the provider must implement +// to implement the base functionality +type ProviderBase interface { + ProviderLifecycle + Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error + Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error +} + // Provider is the interface that the PubSub provider must implement type Provider interface { - ProviderLifecycle + ProviderBase // ID Get the provider ID as specified in the configuration ID() string // TypeID Get the provider type id (e.g. "kafka", "nats") @@ -25,13 +35,13 @@ type Provider interface { } // ProviderBuilder is the interface that the provider builder must implement. -type ProviderBuilder[P, E any] interface { +type ProviderBuilder[P any, E any] interface { // TypeID Get the provider type id (e.g. "kafka", "nats") TypeID() string // BuildProvider Build the provider and the adapter BuildProvider(options P) (Provider, error) // BuildEngineDataSourceFactory Build the data source for the given provider and event configuration - BuildEngineDataSourceFactory(data E) (EngineDataSourceFactory, error) + BuildEngineDataSourceFactory(data E, providers map[string]Provider) (EngineDataSourceFactory, error) } // ProviderType represents the type of pubsub provider @@ -52,6 +62,10 @@ type StreamEvent interface { type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) +type OnPublishEventsFn func(ctx context.Context, pubConf PublishEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) + +type OnStreamEventsFn func(ctx context.Context, subConf SubscriptionEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) + // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { ProviderID() string diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 9e1223d950..ccbec577fa 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -9,7 +9,7 @@ import ( type PubSubProvider struct { id string typeID string - Adapter ProviderLifecycle + Adapter ProviderBase Logger *zap.Logger } @@ -35,7 +35,15 @@ func (p *PubSubProvider) Shutdown(ctx context.Context) error { return nil } -func NewPubSubProvider(id string, typeID string, adapter ProviderLifecycle, logger *zap.Logger) *PubSubProvider { +func (p *PubSubProvider) Subscribe(ctx context.Context, conf SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + return p.Adapter.Subscribe(ctx, conf, updater) +} + +func (p *PubSubProvider) Publish(ctx context.Context, conf PublishEventConfiguration, events []StreamEvent) error { + return p.Adapter.Publish(ctx, conf, events) +} + +func NewPubSubProvider(id string, typeID string, adapter ProviderBase, logger *zap.Logger) *PubSubProvider { return &PubSubProvider{ id: id, typeID: typeID, diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index 6579b62072..134bfbd6bb 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -10,7 +10,7 @@ import ( ) func TestProvider_Startup_Success(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Startup", mock.Anything).Return(nil) provider := PubSubProvider{ @@ -22,7 +22,7 @@ func TestProvider_Startup_Success(t *testing.T) { } func TestProvider_Startup_Error(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Startup", mock.Anything).Return(errors.New("connect error")) provider := PubSubProvider{ @@ -34,7 +34,7 @@ func TestProvider_Startup_Error(t *testing.T) { } func TestProvider_Shutdown_Success(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Shutdown", mock.Anything).Return(nil) provider := PubSubProvider{ @@ -46,7 +46,7 @@ func TestProvider_Shutdown_Success(t *testing.T) { } func TestProvider_Shutdown_Error(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Shutdown", mock.Anything).Return(errors.New("close error")) provider := PubSubProvider{ diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 9332d10f7a..e882a135e7 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -6,7 +6,7 @@ import "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" // that provides a way to send the event struct instead of the raw data // It is used to give access to the event additional fields to the hooks. type SubscriptionEventUpdater interface { - Update(event StreamEvent) + Update(events []StreamEvent) Complete() Close(kind resolve.SubscriptionCloseKind) } @@ -15,8 +15,10 @@ type subscriptionEventUpdater struct { eventUpdater resolve.SubscriptionUpdater } -func (h *subscriptionEventUpdater) Update(event StreamEvent) { - h.eventUpdater.Update(event.GetData()) +func (h *subscriptionEventUpdater) Update(events []StreamEvent) { + for _, event := range events { + h.eventUpdater.Update(event.GetData()) + } } func (h *subscriptionEventUpdater) Complete() { diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 17b805f72e..0d6258d00a 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -18,13 +18,8 @@ var ( errClientClosed = errors.New("client closed") ) -// Adapter defines the interface for Kafka adapter operations -type Adapter interface { - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error - Publish(ctx context.Context, event PublishEventConfiguration) error - Startup(ctx context.Context) error - Shutdown(ctx context.Context) error -} +// Ensure ProviderAdapter implements ProviderBase +var _ datasource.ProviderBase = (*ProviderAdapter)(nil) // ProviderAdapter is a Kafka pubsub implementation. // It uses the franz-go Kafka client to consume and produce messages. @@ -92,11 +87,11 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u headers[header.Key] = header.Value } - updater.Update(&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: r.Value, Headers: headers, Key: r.Key, - }) + }}) } } } @@ -104,23 +99,27 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u // Subscribe subscribes to the given topics and updates the subscription updater. // The engine already deduplicates subscriptions with the same topics, stream configuration, extensions, headers, etc. -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := conf.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", conf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("topics", event.Topics), + zap.Strings("topics", subConf.Topics), ) // Create a new client for the topic client, err := kgo.NewClient(append(p.opts, - kgo.ConsumeTopics(event.Topics...), + kgo.ConsumeTopics(subConf.Topics...), // We want to consume the events produced after the first subscription was created // Messages are shared among all subscriptions, therefore old events are not redelivered // This replicates a stateless publish-subscribe model kgo.ConsumeResetOffset(kgo.NewOffset().AfterMilli(time.Now().UnixMilli())), // For observability, we set the client ID to "router" - kgo.ClientID(fmt.Sprintf("cosmo.router.consumer.%s", strings.Join(event.Topics, "-"))), + kgo.ClientID(fmt.Sprintf("cosmo.router.consumer.%s", strings.Join(subConf.Topics, "-"))), // FIXME: the client id should have some unique identifier, like in nats // What if we have multiple subscriptions for the same topics? // What if we have more router instances? @@ -151,52 +150,71 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return nil } -// Publish publishes the given event to the Kafka topic in a non-blocking way. +// Publish publishes the given events to the Kafka topic in a non-blocking way. // Publish errors are logged and returned as a pubsub error. -// The event is written with a dedicated write client. -func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { +// The events are written with a dedicated write client. +func (p *ProviderAdapter) Publish(ctx context.Context, conf datasource.PublishEventConfiguration, events []datasource.StreamEvent) error { + pubConf, ok := conf.(*PublishEventConfiguration) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", conf.ProviderID()), zap.String("method", "publish"), - zap.String("topic", event.Topic), + zap.String("topic", pubConf.Topic), ) if p.writeClient == nil { return datasource.NewError("kafka write client not initialized", nil) } - log.Debug("publish", zap.ByteString("data", event.Event.Data)) + if len(events) == 0 { + return nil + } + + log.Debug("publish", zap.Int("event_count", len(events))) var wg sync.WaitGroup - wg.Add(1) + wg.Add(len(events)) var pErr error + var errMutex sync.Mutex - headers := make([]kgo.RecordHeader, 0, len(event.Event.Headers)) - for key, value := range event.Event.Headers { - headers = append(headers, kgo.RecordHeader{ - Key: key, - Value: value, - }) - } + for _, streamEvent := range events { + kafkaEvent, ok := streamEvent.(*Event) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } - p.writeClient.Produce(ctx, &kgo.Record{ - Key: event.Event.Key, - Topic: event.Topic, - Value: event.Event.Data, - Headers: headers, - }, func(record *kgo.Record, err error) { - defer wg.Done() - if err != nil { - pErr = err + headers := make([]kgo.RecordHeader, 0, len(kafkaEvent.Headers)) + for key, value := range kafkaEvent.Headers { + headers = append(headers, kgo.RecordHeader{ + Key: key, + Value: value, + }) } - }) + + p.writeClient.Produce(ctx, &kgo.Record{ + Key: kafkaEvent.Key, + Topic: pubConf.Topic, + Value: kafkaEvent.Data, + Headers: headers, + }, func(record *kgo.Record, err error) { + defer wg.Done() + if err != nil { + errMutex.Lock() + pErr = err + errMutex.Unlock() + } + }) + } wg.Wait() if pErr != nil { log.Error("publish error", zap.Error(pErr)) - return datasource.NewError(fmt.Sprintf("error publishing to Kafka topic %s", event.Topic), pErr) + return datasource.NewError(fmt.Sprintf("error publishing to Kafka topic %s", pubConf.Topic), pErr) } return nil diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 2b27e13363..c203a76ac8 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -49,7 +49,7 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { type PublishEventConfiguration struct { Provider string `json:"providerId"` Topic string `json:"topic"` - Event Event `json:"event"` + Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -85,7 +85,7 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { } type SubscriptionDataSource struct { - pubSub Adapter + pubSub datasource.ProviderBase } func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { @@ -128,11 +128,11 @@ func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updat return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) + return s.pubSub.Subscribe(ctx.Context(), conf, updater) } type PublishDataSource struct { - pubSub Adapter + pubSub datasource.ProviderBase } func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { @@ -142,7 +142,7 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B return err } - if err := s.pubSub.Publish(ctx, publishConfiguration); err != nil { + if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index b4672ebfb4..535a4ebb60 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -21,7 +21,7 @@ type EngineDataSourceFactory struct { topics []string providerId string - KafkaAdapter Adapter + KafkaAdapter datasource.ProviderBase } func (c *EngineDataSourceFactory) GetFieldName() string { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go index c1bd6f0d56..b7f7d44d54 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" ) @@ -29,7 +30,7 @@ func TestKafkaEngineDataSourceFactory(t *testing.T) { // TestEngineDataSourceFactoryWithMockAdapter tests the EngineDataSourceFactory with a mocked adapter func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // Create mock adapter - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) // Configure mock expectations for Publish mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { @@ -63,7 +64,7 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // TestEngineDataSourceFactory_GetResolveDataSource_WrongType tests the EngineDataSourceFactory with a mocked adapter func TestEngineDataSourceFactory_GetResolveDataSource_WrongType(t *testing.T) { // Create mock adapter - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) // Create the data source with mock adapter pubsub := &EngineDataSourceFactory{ diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 881b7779e0..317c71a9f4 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -97,7 +97,7 @@ func TestSubscriptionSource_UniqueRequestID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { source := &SubscriptionDataSource{ - pubSub: NewMockAdapter(t), + pubSub: datasource.NewMockProvider(t), } ctx := &resolve.Context{} input := []byte(tt.input) @@ -124,13 +124,13 @@ func TestSubscriptionSource_Start(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) + mockSetup func(*datasource.MockProvider, *datasource.MockSubscriptionEventUpdater) expectError bool }{ { name: "successful subscription", input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1", "topic2"}, @@ -141,7 +141,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "adapter returns error", input: `{"topics":["topic1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1"}, @@ -152,14 +152,14 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) {}, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) updater := datasource.NewMockSubscriptionEventUpdater(t) tt.mockSetup(mockAdapter, updater) @@ -190,7 +190,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter) + mockSetup func(*datasource.MockProvider) expectError bool expectedOutput string expectPublished bool @@ -198,7 +198,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { { name: "successful publish", input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter) { + mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Topic == "test-topic" && @@ -212,7 +212,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { { name: "publish error", input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter) { + mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) }, expectError: false, // The Load method doesn't return the publish error directly @@ -222,7 +222,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter) {}, + mockSetup: func(m *datasource.MockProvider) {}, expectError: true, expectPublished: false, }, @@ -230,7 +230,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) tt.mockSetup(mockAdapter) dataSource := &PublishDataSource{ @@ -255,7 +255,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { func TestKafkaPublishDataSource_LoadWithFiles(t *testing.T) { t.Run("panic on not implemented", func(t *testing.T) { dataSource := &PublishDataSource{ - pubSub: NewMockAdapter(t), + pubSub: datasource.NewMockProvider(t), } assert.Panics(t, func() { diff --git a/router/pkg/pubsub/kafka/mocks.go b/router/pkg/pubsub/kafka/mocks.go index da945393bc..0b917d6d09 100644 --- a/router/pkg/pubsub/kafka/mocks.go +++ b/router/pkg/pubsub/kafka/mocks.go @@ -39,16 +39,16 @@ func (_m *MockAdapter) EXPECT() *MockAdapter_Expecter { } // Publish provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { - ret := _mock.Called(ctx, event) +func (_mock *MockAdapter) Publish(ctx context.Context, event *PublishEventConfiguration, events []datasource.StreamEvent) error { + ret := _mock.Called(ctx, event, events) if len(ret) == 0 { panic("no return value specified for Publish") } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, PublishEventConfiguration) error); ok { - r0 = returnFunc(ctx, event) + if returnFunc, ok := ret.Get(0).(func(context.Context, *PublishEventConfiguration, []datasource.StreamEvent) error); ok { + r0 = returnFunc(ctx, event, events) } else { r0 = ret.Error(0) } @@ -62,24 +62,30 @@ type MockAdapter_Publish_Call struct { // Publish is a helper method to define mock.On call // - ctx context.Context -// - event PublishEventConfiguration -func (_e *MockAdapter_Expecter) Publish(ctx interface{}, event interface{}) *MockAdapter_Publish_Call { - return &MockAdapter_Publish_Call{Call: _e.mock.On("Publish", ctx, event)} +// - event *PublishEventConfiguration +// - events []datasource.StreamEvent +func (_e *MockAdapter_Expecter) Publish(ctx interface{}, event interface{}, events interface{}) *MockAdapter_Publish_Call { + return &MockAdapter_Publish_Call{Call: _e.mock.On("Publish", ctx, event, events)} } -func (_c *MockAdapter_Publish_Call) Run(run func(ctx context.Context, event PublishEventConfiguration)) *MockAdapter_Publish_Call { +func (_c *MockAdapter_Publish_Call) Run(run func(ctx context.Context, event *PublishEventConfiguration, events []datasource.StreamEvent)) *MockAdapter_Publish_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 PublishEventConfiguration + var arg1 *PublishEventConfiguration if args[1] != nil { - arg1 = args[1].(PublishEventConfiguration) + arg1 = args[1].(*PublishEventConfiguration) + } + var arg2 []datasource.StreamEvent + if args[2] != nil { + arg2 = args[2].([]datasource.StreamEvent) } run( arg0, arg1, + arg2, ) }) return _c @@ -90,7 +96,7 @@ func (_c *MockAdapter_Publish_Call) Return(err error) *MockAdapter_Publish_Call return _c } -func (_c *MockAdapter_Publish_Call) RunAndReturn(run func(ctx context.Context, event PublishEventConfiguration) error) *MockAdapter_Publish_Call { +func (_c *MockAdapter_Publish_Call) RunAndReturn(run func(ctx context.Context, event *PublishEventConfiguration, events []datasource.StreamEvent) error) *MockAdapter_Publish_Call { _c.Call.Return(run) return _c } @@ -198,7 +204,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event *SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -206,7 +212,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, *SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -221,21 +227,21 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context -// - event SubscriptionEventConfiguration +// - event *SubscriptionEventConfiguration // - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event *SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 SubscriptionEventConfiguration + var arg1 *SubscriptionEventConfiguration if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) + arg1 = args[1].(*SubscriptionEventConfiguration) } var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { @@ -255,7 +261,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event *SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/kafka/provider_builder.go b/router/pkg/pubsub/kafka/provider_builder.go index 3007b1fafe..1a84d8be51 100644 --- a/router/pkg/pubsub/kafka/provider_builder.go +++ b/router/pkg/pubsub/kafka/provider_builder.go @@ -23,16 +23,15 @@ type ProviderBuilder struct { logger *zap.Logger hostName string routerListenAddr string - adapters map[string]Adapter } func (p *ProviderBuilder) TypeID() string { return providerTypeID } -func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.KafkaEventConfiguration) (datasource.EngineDataSourceFactory, error) { +func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.KafkaEventConfiguration, providers map[string]datasource.Provider) (datasource.EngineDataSourceFactory, error) { providerId := data.GetEngineEventConfiguration().GetProviderId() - adapter, ok := p.adapters[providerId] + provider, ok := providers[providerId] if !ok { return nil, fmt.Errorf("failed to get adapter for provider %s with ID %s", p.TypeID(), providerId) } @@ -52,18 +51,16 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.KafkaEventCo eventType: eventType, topics: data.GetTopics(), providerId: providerId, - KafkaAdapter: adapter, + KafkaAdapter: provider, }, nil } func (p *ProviderBuilder) BuildProvider(provider config.KafkaEventSource) (datasource.Provider, error) { - adapter, pubSubProvider, err := buildProvider(p.ctx, provider, p.logger) + pubSubProvider, err := buildProvider(p.ctx, provider, p.logger) if err != nil { return nil, err } - p.adapters[provider.ID] = adapter - return pubSubProvider, nil } @@ -150,18 +147,18 @@ func buildKafkaOptions(eventSource config.KafkaEventSource, logger *zap.Logger) return opts, nil } -func buildProvider(ctx context.Context, provider config.KafkaEventSource, logger *zap.Logger) (Adapter, datasource.Provider, error) { +func buildProvider(ctx context.Context, provider config.KafkaEventSource, logger *zap.Logger) (datasource.Provider, error) { options, err := buildKafkaOptions(provider, logger) if err != nil { - return nil, nil, fmt.Errorf("failed to build options for Kafka provider with ID \"%s\": %w", provider.ID, err) + return nil, fmt.Errorf("failed to build options for Kafka provider with ID \"%s\": %w", provider.ID, err) } adapter, err := NewProviderAdapter(ctx, logger, options) if err != nil { - return nil, nil, fmt.Errorf("failed to create adapter for Kafka provider with ID \"%s\": %w", provider.ID, err) + return nil, fmt.Errorf("failed to create adapter for Kafka provider with ID \"%s\": %w", provider.ID, err) } pubSubProvider := datasource.NewPubSubProvider(provider.ID, providerTypeID, adapter, logger) - return adapter, pubSubProvider, nil + return pubSubProvider, nil } func NewProviderBuilder( @@ -175,6 +172,5 @@ func NewProviderBuilder( logger: logger, hostName: hostName, routerListenAddr: routerListenAddr, - adapters: make(map[string]Adapter), } } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 4e332c056f..2310eb2469 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -17,18 +17,14 @@ import ( // Adapter defines the methods that a NATS adapter should implement type Adapter interface { - // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error - // Publish publishes the given event to the specified subject - Publish(ctx context.Context, event PublishAndRequestEventConfiguration) error + datasource.ProviderBase // Request sends a request to the specified subject and writes the response to the given writer - Request(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer) error - // Startup initializes the adapter - Startup(ctx context.Context) error - // Shutdown gracefully shuts down the adapter - Shutdown(ctx context.Context) error + Request(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer) error } +// Ensure ProviderAdapter implements ProviderSubscriptionHooks +var _ datasource.ProviderBase = (*ProviderAdapter)(nil) + // ProviderAdapter implements the AdapterInterface for NATS pub/sub type ProviderAdapter struct { ctx context.Context @@ -71,11 +67,16 @@ func (p *ProviderAdapter) getDurableConsumerName(durableName string, subjects [] return fmt.Sprintf("%s-%x", durableName, subjHash.Sum64()), nil } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := cfg.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("subscription event not support by nats provider", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", subConf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("subjects", event.Subjects), + zap.Strings("subjects", subConf.Subjects), ) if p.client == nil { @@ -86,24 +87,24 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return datasource.NewError("nats jetstream not initialized", nil) } - if event.StreamConfiguration != nil { - durableConsumerName, err := p.getDurableConsumerName(event.StreamConfiguration.Consumer, event.Subjects) + if subConf.StreamConfiguration != nil { + durableConsumerName, err := p.getDurableConsumerName(subConf.StreamConfiguration.Consumer, subConf.Subjects) if err != nil { return err } consumerConfig := jetstream.ConsumerConfig{ Durable: durableConsumerName, - FilterSubjects: event.Subjects, + FilterSubjects: subConf.Subjects, } // Durable consumers are removed automatically only if the InactiveThreshold value is set - if event.StreamConfiguration.ConsumerInactiveThreshold > 0 { - consumerConfig.InactiveThreshold = time.Duration(event.StreamConfiguration.ConsumerInactiveThreshold) * time.Second + if subConf.StreamConfiguration.ConsumerInactiveThreshold > 0 { + consumerConfig.InactiveThreshold = time.Duration(subConf.StreamConfiguration.ConsumerInactiveThreshold) * time.Second } - consumer, err := p.js.CreateOrUpdateConsumer(ctx, event.StreamConfiguration.StreamName, consumerConfig) + consumer, err := p.js.CreateOrUpdateConsumer(ctx, subConf.StreamConfiguration.StreamName, consumerConfig) if err != nil { log.Error("creating or updating consumer", zap.Error(err)) - return datasource.NewError(fmt.Sprintf(`failed to create or update consumer for stream "%s"`, event.StreamConfiguration.StreamName), err) + return datasource.NewError(fmt.Sprintf(`failed to create or update consumer for stream "%s"`, subConf.StreamConfiguration.StreamName), err) } p.closeWg.Add(1) @@ -131,10 +132,10 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) - updater.Update(&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data(), Headers: msg.Headers(), - }) + }}) // Acknowledge the message after it has been processed ackErr := msg.Ack() @@ -152,8 +153,8 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } msgChan := make(chan *nats.Msg) - subscriptions := make([]*nats.Subscription, len(event.Subjects)) - for i, subject := range event.Subjects { + subscriptions := make([]*nats.Subscription, len(subConf.Subjects)) + for i, subject := range subConf.Subjects { subscription, err := p.client.ChanSubscribe(subject, msgChan) if err != nil { log.Error("subscribing to NATS subject", zap.Error(err), zap.String("subscription_subject", subject)) @@ -171,10 +172,10 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent select { case msg := <-msgChan: log.Debug("subscription update", zap.String("message_subject", msg.Subject), zap.ByteString("data", msg.Data)) - updater.Update(&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data, Headers: msg.Header, - }) + }}) case <-p.ctx.Done(): // When the application context is done, we stop the subscriptions for _, subscription := range subscriptions { @@ -202,45 +203,71 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return nil } -func (p *ProviderAdapter) Publish(_ context.Context, event PublishAndRequestEventConfiguration) error { +func (p *ProviderAdapter) Publish(_ context.Context, conf datasource.PublishEventConfiguration, events []datasource.StreamEvent) error { + pubConf, ok := conf.(*PublishAndRequestEventConfiguration) + if !ok { + return datasource.NewError("publish event not support by nats provider", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", pubConf.ProviderID()), zap.String("method", "publish"), - zap.String("subject", event.Subject), + zap.String("subject", pubConf.Subject), ) if p.client == nil { return datasource.NewError("nats client not initialized", nil) } - log.Debug("publish", zap.ByteString("data", event.Event.Data)) + if len(events) == 0 { + return nil + } - err := p.client.Publish(event.Subject, event.Event.Data) - if err != nil { - log.Error("publish error", zap.Error(err)) - return datasource.NewError(fmt.Sprintf("error publishing to NATS subject %s", event.Subject), err) + log.Debug("publish", zap.Int("event_count", len(events))) + + for _, streamEvent := range events { + natsEvent, ok := streamEvent.(*Event) + if !ok { + return datasource.NewError("invalid event type for NATS adapter", nil) + } + + err := p.client.Publish(pubConf.Subject, natsEvent.Data) + if err != nil { + log.Error("publish error", zap.Error(err)) + return datasource.NewError(fmt.Sprintf("error publishing to NATS subject %s", pubConf.Subject), err) + } } return nil } -func (p *ProviderAdapter) Request(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer) error { +func (p *ProviderAdapter) Request(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer) error { + reqConf, ok := cfg.(*PublishAndRequestEventConfiguration) + if !ok { + return datasource.NewError("publish event not support by nats provider", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", cfg.ProviderID()), zap.String("method", "request"), - zap.String("subject", event.Subject), + zap.String("subject", reqConf.Subject), ) if p.client == nil { return datasource.NewError("nats client not initialized", nil) } - log.Debug("request", zap.ByteString("data", event.Event.Data)) + natsEvent, ok := event.(*Event) + if !ok { + return datasource.NewError("invalid event type for NATS adapter", nil) + } + + log.Debug("request", zap.ByteString("data", natsEvent.Data)) - msg, err := p.client.RequestWithContext(ctx, event.Subject, event.Event.Data) + msg, err := p.client.RequestWithContext(ctx, reqConf.Subject, natsEvent.Data) if err != nil { log.Error("request error", zap.Error(err)) - return datasource.NewError(fmt.Sprintf("error requesting from NATS subject %s", event.Subject), err) + return datasource.NewError(fmt.Sprintf("error requesting from NATS subject %s", reqConf.Subject), err) } _, err = w.Write(msg.Data) diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index a00f30f4a4..06c2778136 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -55,7 +55,7 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { type PublishAndRequestEventConfiguration struct { Provider string `json:"providerId"` Subject string `json:"subject"` - Event Event `json:"event"` + Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -125,7 +125,7 @@ func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater d return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) + return s.pubSub.Subscribe(ctx.Context(), conf, updater) } type NatsPublishDataSource struct { @@ -139,7 +139,7 @@ func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *byt return err } - if err := s.pubSub.Publish(ctx, publishConfiguration); err != nil { + if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } @@ -162,7 +162,7 @@ func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *byt return err } - return s.pubSub.Request(ctx, subscriptionConfiguration, out) + return s.pubSub.Request(ctx, &subscriptionConfiguration, &subscriptionConfiguration.Event, out) } func (s *NatsRequestDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) error { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index e43c49e8ea..1f4fecd550 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -6,7 +6,6 @@ import ( "slices" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) diff --git a/router/pkg/pubsub/nats/mocks.go b/router/pkg/pubsub/nats/mocks.go index 8c356b7b1c..cfe1a57d95 100644 --- a/router/pkg/pubsub/nats/mocks.go +++ b/router/pkg/pubsub/nats/mocks.go @@ -40,16 +40,16 @@ func (_m *MockAdapter) EXPECT() *MockAdapter_Expecter { } // Publish provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Publish(ctx context.Context, event PublishAndRequestEventConfiguration) error { - ret := _mock.Called(ctx, event) +func (_mock *MockAdapter) Publish(ctx context.Context, cfg datasource.PublishEventConfiguration, events []datasource.StreamEvent) error { + ret := _mock.Called(ctx, cfg, events) if len(ret) == 0 { panic("no return value specified for Publish") } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, PublishAndRequestEventConfiguration) error); ok { - r0 = returnFunc(ctx, event) + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.PublishEventConfiguration, []datasource.StreamEvent) error); ok { + r0 = returnFunc(ctx, cfg, events) } else { r0 = ret.Error(0) } @@ -63,24 +63,30 @@ type MockAdapter_Publish_Call struct { // Publish is a helper method to define mock.On call // - ctx context.Context -// - event PublishAndRequestEventConfiguration -func (_e *MockAdapter_Expecter) Publish(ctx interface{}, event interface{}) *MockAdapter_Publish_Call { - return &MockAdapter_Publish_Call{Call: _e.mock.On("Publish", ctx, event)} +// - cfg datasource.PublishEventConfiguration +// - events []datasource.StreamEvent +func (_e *MockAdapter_Expecter) Publish(ctx interface{}, cfg interface{}, events interface{}) *MockAdapter_Publish_Call { + return &MockAdapter_Publish_Call{Call: _e.mock.On("Publish", ctx, cfg, events)} } -func (_c *MockAdapter_Publish_Call) Run(run func(ctx context.Context, event PublishAndRequestEventConfiguration)) *MockAdapter_Publish_Call { +func (_c *MockAdapter_Publish_Call) Run(run func(ctx context.Context, cfg datasource.PublishEventConfiguration, events []datasource.StreamEvent)) *MockAdapter_Publish_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 PublishAndRequestEventConfiguration + var arg1 datasource.PublishEventConfiguration if args[1] != nil { - arg1 = args[1].(PublishAndRequestEventConfiguration) + arg1 = args[1].(datasource.PublishEventConfiguration) + } + var arg2 []datasource.StreamEvent + if args[2] != nil { + arg2 = args[2].([]datasource.StreamEvent) } run( arg0, arg1, + arg2, ) }) return _c @@ -91,22 +97,22 @@ func (_c *MockAdapter_Publish_Call) Return(err error) *MockAdapter_Publish_Call return _c } -func (_c *MockAdapter_Publish_Call) RunAndReturn(run func(ctx context.Context, event PublishAndRequestEventConfiguration) error) *MockAdapter_Publish_Call { +func (_c *MockAdapter_Publish_Call) RunAndReturn(run func(ctx context.Context, cfg datasource.PublishEventConfiguration, events []datasource.StreamEvent) error) *MockAdapter_Publish_Call { _c.Call.Return(run) return _c } // Request provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Request(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer) error { - ret := _mock.Called(ctx, event, w) +func (_mock *MockAdapter) Request(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer) error { + ret := _mock.Called(ctx, cfg, event, w) if len(ret) == 0 { panic("no return value specified for Request") } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, PublishAndRequestEventConfiguration, io.Writer) error); ok { - r0 = returnFunc(ctx, event, w) + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.PublishEventConfiguration, datasource.StreamEvent, io.Writer) error); ok { + r0 = returnFunc(ctx, cfg, event, w) } else { r0 = ret.Error(0) } @@ -120,30 +126,36 @@ type MockAdapter_Request_Call struct { // Request is a helper method to define mock.On call // - ctx context.Context -// - event PublishAndRequestEventConfiguration +// - cfg datasource.PublishEventConfiguration +// - event datasource.StreamEvent // - w io.Writer -func (_e *MockAdapter_Expecter) Request(ctx interface{}, event interface{}, w interface{}) *MockAdapter_Request_Call { - return &MockAdapter_Request_Call{Call: _e.mock.On("Request", ctx, event, w)} +func (_e *MockAdapter_Expecter) Request(ctx interface{}, cfg interface{}, event interface{}, w interface{}) *MockAdapter_Request_Call { + return &MockAdapter_Request_Call{Call: _e.mock.On("Request", ctx, cfg, event, w)} } -func (_c *MockAdapter_Request_Call) Run(run func(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer)) *MockAdapter_Request_Call { +func (_c *MockAdapter_Request_Call) Run(run func(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer)) *MockAdapter_Request_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 PublishAndRequestEventConfiguration + var arg1 datasource.PublishEventConfiguration if args[1] != nil { - arg1 = args[1].(PublishAndRequestEventConfiguration) + arg1 = args[1].(datasource.PublishEventConfiguration) } - var arg2 io.Writer + var arg2 datasource.StreamEvent if args[2] != nil { - arg2 = args[2].(io.Writer) + arg2 = args[2].(datasource.StreamEvent) + } + var arg3 io.Writer + if args[3] != nil { + arg3 = args[3].(io.Writer) } run( arg0, arg1, arg2, + arg3, ) }) return _c @@ -154,7 +166,7 @@ func (_c *MockAdapter_Request_Call) Return(err error) *MockAdapter_Request_Call return _c } -func (_c *MockAdapter_Request_Call) RunAndReturn(run func(ctx context.Context, event PublishAndRequestEventConfiguration, w io.Writer) error) *MockAdapter_Request_Call { +func (_c *MockAdapter_Request_Call) RunAndReturn(run func(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer) error) *MockAdapter_Request_Call { _c.Call.Return(run) return _c } @@ -262,16 +274,16 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { - ret := _mock.Called(ctx, event, updater) +func (_mock *MockAdapter) Subscribe(ctx context.Context, cfg datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + ret := _mock.Called(ctx, cfg, updater) if len(ret) == 0 { panic("no return value specified for Subscribe") } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { - r0 = returnFunc(ctx, event, updater) + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { + r0 = returnFunc(ctx, cfg, updater) } else { r0 = ret.Error(0) } @@ -285,21 +297,21 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context -// - event SubscriptionEventConfiguration +// - cfg datasource.SubscriptionEventConfiguration // - updater datasource.SubscriptionEventUpdater -func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { - return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} +func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, cfg interface{}, updater interface{}) *MockAdapter_Subscribe_Call { + return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, cfg, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, cfg datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 SubscriptionEventConfiguration + var arg1 datasource.SubscriptionEventConfiguration if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) + arg1 = args[1].(datasource.SubscriptionEventConfiguration) } var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { @@ -319,7 +331,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, cfg datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/nats/provider_builder.go b/router/pkg/pubsub/nats/provider_builder.go index e1ae0a1e70..b1cf1a21d7 100644 --- a/router/pkg/pubsub/nats/provider_builder.go +++ b/router/pkg/pubsub/nats/provider_builder.go @@ -27,13 +27,18 @@ func (p *ProviderBuilder) TypeID() string { return providerTypeID } -func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventConfiguration) (datasource.EngineDataSourceFactory, error) { +func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventConfiguration, providers map[string]datasource.Provider) (datasource.EngineDataSourceFactory, error) { providerId := data.GetEngineEventConfiguration().GetProviderId() - adapter, ok := p.adapters[providerId] + provider, ok := providers[providerId] if !ok { return nil, fmt.Errorf("failed to get adapter for provider %s with ID %s", p.TypeID(), providerId) } + adapter, ok := provider.(Adapter) + if !ok { + return nil, fmt.Errorf("adapter for provider %s is not of the right type", providerId) + } + var eventType EventType switch data.GetEngineEventConfiguration().GetType() { case nodev1.EventType_PUBLISH: @@ -65,10 +70,14 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventCon } func (p *ProviderBuilder) BuildProvider(provider config.NatsEventSource) (datasource.Provider, error) { - adapter, pubSubProvider, err := buildProvider(p.ctx, provider, p.logger, p.hostName, p.routerListenAddr) + adapterBase, pubSubProvider, err := buildProvider(p.ctx, provider, p.logger, p.hostName, p.routerListenAddr) if err != nil { return nil, err } + adapter, ok := adapterBase.(Adapter) + if !ok { + return nil, fmt.Errorf("adapter for provider %s is not an Adapter", provider.ID) + } p.adapters[provider.ID] = adapter return pubSubProvider, nil @@ -118,7 +127,7 @@ func buildNatsOptions(eventSource config.NatsEventSource, logger *zap.Logger) ([ return opts, nil } -func buildProvider(ctx context.Context, provider config.NatsEventSource, logger *zap.Logger, hostName string, routerListenAddr string) (Adapter, datasource.Provider, error) { +func buildProvider(ctx context.Context, provider config.NatsEventSource, logger *zap.Logger, hostName string, routerListenAddr string) (datasource.ProviderBase, datasource.Provider, error) { options, err := buildNatsOptions(provider, logger) if err != nil { return nil, nil, fmt.Errorf("failed to build options for Nats provider with ID \"%s\": %w", provider.ID, err) diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index f8901e3a50..21405a470b 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -51,6 +51,8 @@ func (e *ProviderNotDefinedError) Error() string { type Hooks struct { SubscriptionOnStart []pubsub_datasource.SubscriptionOnStartFn + OnStreamEvents []pubsub_datasource.OnStreamEventsFn + OnPublishEvents []pubsub_datasource.OnPublishEventsFn } // BuildProvidersAndDataSources is a generic function that builds providers and data sources for the given @@ -80,7 +82,9 @@ func BuildProvidersAndDataSources( if err != nil { return nil, nil, err } - pubSubProviders = append(pubSubProviders, kafkaPubSubProviders...) + for _, provider := range kafkaPubSubProviders { + pubSubProviders = append(pubSubProviders, provider) + } outs = append(outs, kafkaOuts...) // initialize NATS providers and data sources @@ -96,7 +100,9 @@ func BuildProvidersAndDataSources( if err != nil { return nil, nil, err } - pubSubProviders = append(pubSubProviders, natsPubSubProviders...) + for _, provider := range natsPubSubProviders { + pubSubProviders = append(pubSubProviders, provider) + } outs = append(outs, natsOuts...) // initialize Redis providers and data sources @@ -112,14 +118,16 @@ func BuildProvidersAndDataSources( if err != nil { return nil, nil, err } - pubSubProviders = append(pubSubProviders, redisPubSubProviders...) + for _, provider := range redisPubSubProviders { + pubSubProviders = append(pubSubProviders, provider) + } outs = append(outs, redisOuts...) return pubSubProviders, outs, nil } -func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E], hooks Hooks) ([]pubsub_datasource.Provider, []plan.DataSource, error) { - var pubSubProviders []pubsub_datasource.Provider +func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E], hooks Hooks) (map[string]pubsub_datasource.Provider, []plan.DataSource, error) { + pubSubProviders := make(map[string]pubsub_datasource.Provider) var outs []plan.DataSource // check used providers @@ -133,7 +141,6 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder } // initialize providers if used - providerIds := []string{} for _, providerData := range providersData { if !slices.Contains(usedProviderIds, providerData.GetID()) { continue @@ -142,13 +149,16 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder if err != nil { return nil, nil, err } - pubSubProviders = append(pubSubProviders, provider) - providerIds = append(providerIds, provider.ID()) + pubSubProviders[provider.ID()] = pubsub_datasource.NewHookedProvider( + provider, + hooks.OnStreamEvents, + hooks.OnPublishEvents, + ) } // check if all used providers are initialized for _, providerId := range usedProviderIds { - if !slices.Contains(providerIds, providerId) { + if _, ok := pubSubProviders[providerId]; !ok { return pubSubProviders, nil, &ProviderNotDefinedError{ ProviderID: providerId, ProviderTypeID: builder.TypeID(), @@ -159,7 +169,12 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder // build data sources for each event for _, dsConf := range dsConfs { for i, event := range dsConf.events { - plannerConfig := pubsub_datasource.NewPlannerConfig(builder, event, hooks.SubscriptionOnStart) + plannerConfig := pubsub_datasource.NewPlannerConfig( + builder, + event, + pubSubProviders, + hooks.SubscriptionOnStart, + ) out, err := plan.NewDataSourceConfiguration( dsConf.dsConf.Configuration.Id+"-"+builder.TypeID()+"-"+strconv.Itoa(i), pubsub_datasource.NewPlannerFactory(ctx, plannerConfig), diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 556e676048..a3e302fba2 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -13,15 +13,18 @@ import ( // Adapter defines the methods that a Redis adapter should implement type Adapter interface { // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error - // Publish publishes the given event to the specified channel - Publish(ctx context.Context, event PublishEventConfiguration) error + Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error + // Publish publishes the given events to the specified channel + Publish(ctx context.Context, conf datasource.PublishEventConfiguration, events []datasource.StreamEvent) error // Startup initializes the adapter Startup(ctx context.Context) error // Shutdown gracefully shuts down the adapter Shutdown(ctx context.Context) error } +// Ensure ProviderAdapter implements ProviderSubscriptionHooks +var _ datasource.ProviderBase = (*ProviderAdapter)(nil) + func NewProviderAdapter(ctx context.Context, logger *zap.Logger, urls []string, clusterEnabled bool) Adapter { ctx, cancel := context.WithCancel(ctx) return &ProviderAdapter{ @@ -73,19 +76,24 @@ func (p *ProviderAdapter) Shutdown(ctx context.Context) error { return p.conn.Close() } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := conf.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("subscription event not support by redis provider", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", conf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("channels", event.Channels), + zap.Strings("channels", subConf.Channels), ) - sub := p.conn.PSubscribe(ctx, event.Channels...) + sub := p.conn.PSubscribe(ctx, subConf.Channels...) msgChan := sub.Channel() cleanup := func() { - err := sub.PUnsubscribe(ctx, event.Channels...) + err := sub.PUnsubscribe(ctx, subConf.Channels...) if err != nil { - log.Error(fmt.Sprintf("error unsubscribing from redis for topics %v", event.Channels), zap.Error(err)) + log.Error(fmt.Sprintf("error unsubscribing from redis for topics %v", subConf.Channels), zap.Error(err)) } } @@ -106,9 +114,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return } log.Debug("subscription update", zap.String("message_channel", msg.Channel), zap.String("data", msg.Payload)) - updater.Update(&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: []byte(msg.Payload), - }) + }}) case <-p.ctx.Done(): // When the application context is done, we stop the subscription if it is not already done log.Debug("application context done, stopping subscription") @@ -126,27 +134,45 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return nil } -func (p *ProviderAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { +func (p *ProviderAdapter) Publish(ctx context.Context, conf datasource.PublishEventConfiguration, events []datasource.StreamEvent) error { + pubConf, ok := conf.(*PublishEventConfiguration) + if !ok { + return datasource.NewError("publish event not support by redis provider", nil) + } + log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", conf.ProviderID()), zap.String("method", "publish"), - zap.String("channel", event.Channel), + zap.String("channel", pubConf.Channel), ) - log.Debug("publish", zap.ByteString("data", event.Event.Data)) - - data, dataErr := event.Event.Data.MarshalJSON() - if dataErr != nil { - log.Error("error marshalling data", zap.Error(dataErr)) - return datasource.NewError("error marshalling data", dataErr) - } if p.conn == nil { return datasource.NewError("redis connection not initialized", nil) } - intCmd := p.conn.Publish(ctx, event.Channel, data) - if intCmd.Err() != nil { - log.Error("publish error", zap.Error(intCmd.Err())) - return datasource.NewError(fmt.Sprintf("error publishing to Redis PubSub channel %s", event.Channel), intCmd.Err()) + + if len(events) == 0 { + return nil + } + + log.Debug("publish", zap.Int("event_count", len(events))) + + for _, streamEvent := range events { + redisEvent, ok := streamEvent.(*Event) + if !ok { + return datasource.NewError("invalid event type for Redis adapter", nil) + } + + data, dataErr := redisEvent.Data.MarshalJSON() + if dataErr != nil { + log.Error("error marshalling data", zap.Error(dataErr)) + return datasource.NewError("error marshalling data", dataErr) + } + + intCmd := p.conn.Publish(ctx, pubConf.Channel, data) + if intCmd.Err() != nil { + log.Error("publish error", zap.Error(intCmd.Err())) + return datasource.NewError(fmt.Sprintf("error publishing to Redis PubSub channel %s", pubConf.Channel), intCmd.Err()) + } } return nil diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index cb97bafb9a..a46f41c6be 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -49,7 +49,7 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { type PublishEventConfiguration struct { Provider string `json:"providerId"` Channel string `json:"channel"` - Event Event `json:"event"` + Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -119,7 +119,7 @@ func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updat return fmt.Errorf("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) + return s.pubSub.Subscribe(ctx.Context(), conf, updater) } // LoadInitialData implements the interface method (not used for this subscription type) @@ -140,7 +140,7 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B return err } - if err := s.pubSub.Publish(ctx, publishConfiguration); err != nil { + if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } diff --git a/router/pkg/pubsub/redis/engine_datasource_factory_test.go b/router/pkg/pubsub/redis/engine_datasource_factory_test.go index 3d7910cf23..6724e97f84 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" ) @@ -29,7 +30,7 @@ func TestRedisEngineDataSourceFactory(t *testing.T) { // TestEngineDataSourceFactoryWithMockAdapter tests the EngineDataSourceFactory with a mocked adapter func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // Create mock adapter - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) // Configure mock expectations for Publish mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { @@ -63,7 +64,7 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { // TestEngineDataSourceFactory_GetResolveDataSource_WrongType tests the EngineDataSourceFactory with a mocked adapter func TestEngineDataSourceFactory_GetResolveDataSource_WrongType(t *testing.T) { // Create mock adapter - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) // Create the data source with mock adapter pubsub := &EngineDataSourceFactory{ diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index a343e51503..8d8ee54159 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -79,7 +79,7 @@ func TestSubscriptionSource_UniqueRequestID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { source := &SubscriptionDataSource{ - pubSub: NewMockAdapter(t), + pubSub: datasource.NewMockProvider(t), } ctx := &resolve.Context{} input := []byte(tt.input) @@ -106,13 +106,13 @@ func TestSubscriptionSource_Start(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) + mockSetup func(*datasource.MockProvider, *datasource.MockSubscriptionEventUpdater) expectError bool }{ { name: "successful subscription", input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1", "channel2"}, @@ -123,7 +123,7 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "adapter returns error", input: `{"channels":["channel1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1"}, @@ -134,14 +134,14 @@ func TestSubscriptionSource_Start(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, + mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) {}, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) updater := datasource.NewMockSubscriptionEventUpdater(t) tt.mockSetup(mockAdapter, updater) @@ -172,7 +172,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { tests := []struct { name string input string - mockSetup func(*MockAdapter) + mockSetup func(*datasource.MockProvider) expectError bool expectedOutput string expectPublished bool @@ -180,7 +180,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { { name: "successful publish", input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter) { + mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Channel == "test-channel" && @@ -194,7 +194,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { { name: "publish error", input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter) { + mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) }, expectError: false, // The Load method doesn't return the publish error directly @@ -204,7 +204,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { { name: "invalid input json", input: `{"invalid json":`, - mockSetup: func(m *MockAdapter) {}, + mockSetup: func(m *datasource.MockProvider) {}, expectError: true, expectPublished: false, }, @@ -212,7 +212,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) + mockAdapter := datasource.NewMockProvider(t) tt.mockSetup(mockAdapter) dataSource := &PublishDataSource{ @@ -237,7 +237,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { func TestRedisPublishDataSource_LoadWithFiles(t *testing.T) { t.Run("panic on not implemented", func(t *testing.T) { dataSource := &PublishDataSource{ - pubSub: NewMockAdapter(t), + pubSub: datasource.NewMockProvider(t), } assert.Panics(t, func() { diff --git a/router/pkg/pubsub/redis/mocks.go b/router/pkg/pubsub/redis/mocks.go deleted file mode 100644 index 91c6ca9205..0000000000 --- a/router/pkg/pubsub/redis/mocks.go +++ /dev/null @@ -1,261 +0,0 @@ -// Code generated by mockery; DO NOT EDIT. -// github.com/vektra/mockery -// template: testify - -package redis - -import ( - "context" - - mock "github.com/stretchr/testify/mock" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" -) - -// NewMockAdapter creates a new instance of MockAdapter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewMockAdapter(t interface { - mock.TestingT - Cleanup(func()) -}) *MockAdapter { - mock := &MockAdapter{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// MockAdapter is an autogenerated mock type for the Adapter type -type MockAdapter struct { - mock.Mock -} - -type MockAdapter_Expecter struct { - mock *mock.Mock -} - -func (_m *MockAdapter) EXPECT() *MockAdapter_Expecter { - return &MockAdapter_Expecter{mock: &_m.Mock} -} - -// Publish provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Publish(ctx context.Context, event PublishEventConfiguration) error { - ret := _mock.Called(ctx, event) - - if len(ret) == 0 { - panic("no return value specified for Publish") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, PublishEventConfiguration) error); ok { - r0 = returnFunc(ctx, event) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockAdapter_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' -type MockAdapter_Publish_Call struct { - *mock.Call -} - -// Publish is a helper method to define mock.On call -// - ctx context.Context -// - event PublishEventConfiguration -func (_e *MockAdapter_Expecter) Publish(ctx interface{}, event interface{}) *MockAdapter_Publish_Call { - return &MockAdapter_Publish_Call{Call: _e.mock.On("Publish", ctx, event)} -} - -func (_c *MockAdapter_Publish_Call) Run(run func(ctx context.Context, event PublishEventConfiguration)) *MockAdapter_Publish_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 PublishEventConfiguration - if args[1] != nil { - arg1 = args[1].(PublishEventConfiguration) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *MockAdapter_Publish_Call) Return(err error) *MockAdapter_Publish_Call { - _c.Call.Return(err) - return _c -} - -func (_c *MockAdapter_Publish_Call) RunAndReturn(run func(ctx context.Context, event PublishEventConfiguration) error) *MockAdapter_Publish_Call { - _c.Call.Return(run) - return _c -} - -// Shutdown provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Shutdown(ctx context.Context) error { - ret := _mock.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for Shutdown") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = returnFunc(ctx) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockAdapter_Shutdown_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Shutdown' -type MockAdapter_Shutdown_Call struct { - *mock.Call -} - -// Shutdown is a helper method to define mock.On call -// - ctx context.Context -func (_e *MockAdapter_Expecter) Shutdown(ctx interface{}) *MockAdapter_Shutdown_Call { - return &MockAdapter_Shutdown_Call{Call: _e.mock.On("Shutdown", ctx)} -} - -func (_c *MockAdapter_Shutdown_Call) Run(run func(ctx context.Context)) *MockAdapter_Shutdown_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockAdapter_Shutdown_Call) Return(err error) *MockAdapter_Shutdown_Call { - _c.Call.Return(err) - return _c -} - -func (_c *MockAdapter_Shutdown_Call) RunAndReturn(run func(ctx context.Context) error) *MockAdapter_Shutdown_Call { - _c.Call.Return(run) - return _c -} - -// Startup provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Startup(ctx context.Context) error { - ret := _mock.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for Startup") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = returnFunc(ctx) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockAdapter_Startup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Startup' -type MockAdapter_Startup_Call struct { - *mock.Call -} - -// Startup is a helper method to define mock.On call -// - ctx context.Context -func (_e *MockAdapter_Expecter) Startup(ctx interface{}) *MockAdapter_Startup_Call { - return &MockAdapter_Startup_Call{Call: _e.mock.On("Startup", ctx)} -} - -func (_c *MockAdapter_Startup_Call) Run(run func(ctx context.Context)) *MockAdapter_Startup_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockAdapter_Startup_Call) Return(err error) *MockAdapter_Startup_Call { - _c.Call.Return(err) - return _c -} - -func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) error) *MockAdapter_Startup_Call { - _c.Call.Return(run) - return _c -} - -// Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { - ret := _mock.Called(ctx, event, updater) - - if len(ret) == 0 { - panic("no return value specified for Subscribe") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { - r0 = returnFunc(ctx, event, updater) - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockAdapter_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe' -type MockAdapter_Subscribe_Call struct { - *mock.Call -} - -// Subscribe is a helper method to define mock.On call -// - ctx context.Context -// - event SubscriptionEventConfiguration -// - updater datasource.SubscriptionEventUpdater -func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { - return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} -} - -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 SubscriptionEventConfiguration - if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) - } - var arg2 datasource.SubscriptionEventUpdater - if args[2] != nil { - arg2 = args[2].(datasource.SubscriptionEventUpdater) - } - run( - arg0, - arg1, - arg2, - ) - }) - return _c -} - -func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_Call { - _c.Call.Return(err) - return _c -} - -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { - _c.Call.Return(run) - return _c -} diff --git a/router/pkg/pubsub/redis/provider_builder.go b/router/pkg/pubsub/redis/provider_builder.go index 415963b885..ea643f1c3c 100644 --- a/router/pkg/pubsub/redis/provider_builder.go +++ b/router/pkg/pubsub/redis/provider_builder.go @@ -18,7 +18,6 @@ type ProviderBuilder struct { logger *zap.Logger hostName string routerListenAddr string - adapters map[string]Adapter } // NewProviderBuilder creates a new Redis PubSub provider builder @@ -33,7 +32,6 @@ func NewProviderBuilder( logger: logger, hostName: hostName, routerListenAddr: routerListenAddr, - adapters: make(map[string]Adapter), } } @@ -43,8 +41,12 @@ func (b *ProviderBuilder) TypeID() string { } // DataSource creates a Redis PubSub data source for the given event configuration -func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventConfiguration) (datasource.EngineDataSourceFactory, error) { +func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventConfiguration, providers map[string]datasource.Provider) (datasource.EngineDataSourceFactory, error) { providerId := data.GetEngineEventConfiguration().GetProviderId() + provider, ok := providers[providerId] + if !ok { + return nil, fmt.Errorf("failed to get adapter for provider %s with ID %s", b.TypeID(), providerId) + } var eventType EventType switch data.GetEngineEventConfiguration().GetType() { @@ -57,11 +59,11 @@ func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventCo } return &EngineDataSourceFactory{ - RedisAdapter: b.adapters[providerId], fieldName: data.GetEngineEventConfiguration().GetFieldName(), eventType: eventType, channels: data.GetChannels(), providerId: providerId, + RedisAdapter: provider, }, nil } @@ -69,7 +71,6 @@ func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventCo func (b *ProviderBuilder) BuildProvider(provider config.RedisEventSource) (datasource.Provider, error) { adapter := NewProviderAdapter(b.ctx, b.logger, provider.URLs, provider.ClusterEnabled) pubSubProvider := datasource.NewPubSubProvider(provider.ID, providerTypeID, adapter, b.logger) - b.adapters[provider.ID] = adapter return pubSubProvider, nil } From f7d9b42f8edd0b4860dab5a60902b3c878f12478 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 18:00:15 +0200 Subject: [PATCH 080/173] chore: fixed behaviour when no modules are specified --- .../pkg/pubsub/datasource/hookedprovider.go | 11 ++++++++- .../kafka/engine_datasource_factory_test.go | 5 +++- .../pubsub/kafka/engine_datasource_test.go | 11 +++++---- router/pkg/pubsub/nats/engine_datasource.go | 23 +++++++++++++++---- .../pubsub/nats/engine_datasource_factory.go | 2 +- .../nats/engine_datasource_factory_test.go | 12 +++++++--- .../pkg/pubsub/nats/engine_datasource_test.go | 19 +++++++++------ router/pkg/pubsub/nats/provider_builder.go | 7 +----- .../redis/engine_datasource_factory_test.go | 5 +++- .../pubsub/redis/engine_datasource_test.go | 11 +++++---- 10 files changed, 74 insertions(+), 32 deletions(-) diff --git a/router/pkg/pubsub/datasource/hookedprovider.go b/router/pkg/pubsub/datasource/hookedprovider.go index 2518dcf1e1..a9f8186473 100644 --- a/router/pkg/pubsub/datasource/hookedprovider.go +++ b/router/pkg/pubsub/datasource/hookedprovider.go @@ -16,11 +16,16 @@ type hookedUpdater struct { func (h *hookedUpdater) Update(events []StreamEvent) { var newEvents []StreamEvent var err error + if len(h.OnStreamEventsFns) == 0 { + h.updater.Update(events) + return + } + for _, fn := range h.OnStreamEventsFns { newEvents, err = fn(h.ctx, h.subscriptionEventConfiguration, events) if err != nil { // TODO: do something with the error - return + continue } } @@ -63,6 +68,10 @@ func (h *HookedProvider) Subscribe(ctx context.Context, cfg SubscriptionEventCon func (h *HookedProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { var newEvents []StreamEvent var err error + if len(h.OnPublishEventsFns) == 0 { + return h.Provider.Publish(ctx, cfg, events) + } + for _, fn := range h.OnPublishEventsFns { newEvents, err = fn(ctx, cfg, events) if err != nil { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go index b7f7d44d54..22e5e36f49 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "strings" "testing" "github.com/stretchr/testify/mock" @@ -33,8 +34,10 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { mockAdapter := datasource.NewMockProvider(t) // Configure mock expectations for Publish - mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { + mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Topic == "test-topic" + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"test":"data"}`) })).Return(nil) // Create the data source with mock adapter diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 317c71a9f4..0067859e18 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "strings" "testing" "github.com/cespare/xxhash/v2" @@ -131,7 +132,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "successful subscription", input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1", "topic2"}, }, mock.Anything).Return(nil) @@ -142,7 +143,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "adapter returns error", input: `{"topics":["topic1"], "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Topics: []string{"topic1"}, }, mock.Anything).Return(errors.New("subscription error")) @@ -199,10 +200,12 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { name: "successful publish", input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider) { - m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { + m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Topic == "test-topic" && string(event.Event.Data) == `{"message":"hello"}` + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) }, expectError: false, @@ -213,7 +216,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { name: "publish error", input: `{"topic":"test-topic", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider) { - m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) + m.On("Publish", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("publish error")) }, expectError: false, // The Load method doesn't return the publish error directly expectedOutput: `{"success": false}`, diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 06c2778136..57a92923aa 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -81,7 +81,7 @@ func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() (string, err } type SubscriptionSource struct { - pubSub Adapter + pubSub datasource.ProviderBase } func (s *SubscriptionSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { @@ -129,7 +129,7 @@ func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater d } type NatsPublishDataSource struct { - pubSub Adapter + pubSub datasource.ProviderBase } func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { @@ -152,7 +152,7 @@ func (s *NatsPublishDataSource) LoadWithFiles(ctx context.Context, input []byte, } type NatsRequestDataSource struct { - pubSub Adapter + pubSub datasource.ProviderBase } func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { @@ -162,7 +162,22 @@ func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *byt return err } - return s.pubSub.Request(ctx, &subscriptionConfiguration, &subscriptionConfiguration.Event, out) + hookedProvider, ok := s.pubSub.(*datasource.HookedProvider) + if !ok { + return fmt.Errorf("adapter for provider %s is not of the right hooked type", subscriptionConfiguration.ProviderID()) + } + + providerBase, ok := hookedProvider.Provider.(*datasource.PubSubProvider) + if !ok { + return fmt.Errorf("adapter for provider %s is not of the right type", subscriptionConfiguration.ProviderID()) + } + + adapter, ok := providerBase.Adapter.(Adapter) + if !ok { + return fmt.Errorf("adapter for provider %s is not of the right type", subscriptionConfiguration.ProviderID()) + } + + return adapter.Request(ctx, &subscriptionConfiguration, &subscriptionConfiguration.Event, out) } func (s *NatsRequestDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) error { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 1f4fecd550..68dea2a8e6 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -18,7 +18,7 @@ const ( ) type EngineDataSourceFactory struct { - NatsAdapter Adapter + NatsAdapter datasource.ProviderBase fieldName string eventType EventType diff --git a/router/pkg/pubsub/nats/engine_datasource_factory_test.go b/router/pkg/pubsub/nats/engine_datasource_factory_test.go index 50e20a98e7..8eb28d473f 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory_test.go @@ -5,11 +5,13 @@ import ( "context" "encoding/json" "io" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" ) @@ -33,8 +35,10 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { mockAdapter := NewMockAdapter(t) // Configure mock expectations for Publish - mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { + mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Subject == "test-subject" + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"test":"data"}`) })).Return(nil) // Create the data source with mock adapter @@ -166,10 +170,12 @@ func TestEngineDataSourceFactory_RequestDataSource(t *testing.T) { mockAdapter := NewMockAdapter(t) // Configure mock expectations for Request - mockAdapter.On("Request", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { + mockAdapter.On("Request", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Subject == "test-subject" + }), mock.MatchedBy(func(event datasource.StreamEvent) bool { + return event != nil && strings.EqualFold(string(event.GetData()), `{"test":"data"}`) }), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - w := args.Get(2).(io.Writer) + w := args.Get(3).(io.Writer) w.Write([]byte(`{"response": "test"}`)) }) diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 79b8219e53..2c6609b87e 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "strings" "testing" "github.com/cespare/xxhash/v2" @@ -114,7 +115,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "successful subscription", input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Subjects: []string{"subject1", "subject2"}, }, mock.Anything).Return(nil) @@ -125,7 +126,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "adapter returns error", input: `{"subjects":["subject1"], "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Subjects: []string{"subject1"}, }, mock.Anything).Return(errors.New("subscription error")) @@ -182,10 +183,12 @@ func TestNatsPublishDataSource_Load(t *testing.T) { name: "successful publish", input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { - m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { + m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Subject == "test-subject" && string(event.Event.Data) == `{"message":"hello"}` + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) }, expectError: false, @@ -196,7 +199,7 @@ func TestNatsPublishDataSource_Load(t *testing.T) { name: "publish error", input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { - m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) + m.On("Publish", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("publish error")) }, expectError: false, // The Load method doesn't return the publish error directly expectedOutput: `{"success": false}`, @@ -257,13 +260,15 @@ func TestNatsRequestDataSource_Load(t *testing.T) { name: "successful request", input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { - m.On("Request", mock.Anything, mock.MatchedBy(func(event PublishAndRequestEventConfiguration) bool { + m.On("Request", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Subject == "test-subject" && string(event.Event.Data) == `{"message":"hello"}` + }), mock.MatchedBy(func(event datasource.StreamEvent) bool { + return event != nil && strings.EqualFold(string(event.GetData()), `{"message":"hello"}`) }), mock.Anything).Run(func(args mock.Arguments) { // Write response to the output buffer - w := args.Get(2).(io.Writer) + w := args.Get(3).(io.Writer) _, _ = w.Write([]byte(`{"response":"success"}`)) }).Return(nil) }, @@ -274,7 +279,7 @@ func TestNatsRequestDataSource_Load(t *testing.T) { name: "request error", input: `{"subject":"test-subject", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *MockAdapter) { - m.On("Request", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("request error")) + m.On("Request", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("request error")) }, expectError: true, expectedOutput: "", diff --git a/router/pkg/pubsub/nats/provider_builder.go b/router/pkg/pubsub/nats/provider_builder.go index b1cf1a21d7..d3e791bd74 100644 --- a/router/pkg/pubsub/nats/provider_builder.go +++ b/router/pkg/pubsub/nats/provider_builder.go @@ -34,11 +34,6 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventCon return nil, fmt.Errorf("failed to get adapter for provider %s with ID %s", p.TypeID(), providerId) } - adapter, ok := provider.(Adapter) - if !ok { - return nil, fmt.Errorf("adapter for provider %s is not of the right type", providerId) - } - var eventType EventType switch data.GetEngineEventConfiguration().GetType() { case nodev1.EventType_PUBLISH: @@ -51,7 +46,7 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventCon return nil, fmt.Errorf("unsupported event type: %s", data.GetEngineEventConfiguration().GetType()) } dataSourceFactory := &EngineDataSourceFactory{ - NatsAdapter: adapter, + NatsAdapter: provider, fieldName: data.GetEngineEventConfiguration().GetFieldName(), eventType: eventType, subjects: data.GetSubjects(), diff --git a/router/pkg/pubsub/redis/engine_datasource_factory_test.go b/router/pkg/pubsub/redis/engine_datasource_factory_test.go index 6724e97f84..6a514af216 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "strings" "testing" "github.com/stretchr/testify/mock" @@ -33,8 +34,10 @@ func TestEngineDataSourceFactoryWithMockAdapter(t *testing.T) { mockAdapter := datasource.NewMockProvider(t) // Configure mock expectations for Publish - mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { + mockAdapter.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Channel == "test-channel" + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"test":"data"}`) })).Return(nil) // Create the data source with mock adapter diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 8d8ee54159..8c4859d36c 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "strings" "testing" "github.com/cespare/xxhash/v2" @@ -113,7 +114,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "successful subscription", input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1", "channel2"}, }, mock.Anything).Return(nil) @@ -124,7 +125,7 @@ func TestSubscriptionSource_Start(t *testing.T) { name: "adapter returns error", input: `{"channels":["channel1"], "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ + m.On("Subscribe", mock.Anything, &SubscriptionEventConfiguration{ Provider: "test-provider", Channels: []string{"channel1"}, }, mock.Anything).Return(errors.New("subscription error")) @@ -181,10 +182,12 @@ func TestRedisPublishDataSource_Load(t *testing.T) { name: "successful publish", input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider) { - m.On("Publish", mock.Anything, mock.MatchedBy(func(event PublishEventConfiguration) bool { + m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && event.Channel == "test-channel" && string(event.Event.Data) == `{"message":"hello"}` + }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { + return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) }, expectError: false, @@ -195,7 +198,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { name: "publish error", input: `{"channel":"test-channel", "event": {"data":{"message":"hello"}}, "providerId":"test-provider"}`, mockSetup: func(m *datasource.MockProvider) { - m.On("Publish", mock.Anything, mock.Anything).Return(errors.New("publish error")) + m.On("Publish", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("publish error")) }, expectError: false, // The Load method doesn't return the publish error directly expectedOutput: `{"success": false}`, From 742d6aba389f9f6247ea8885297a79a585d2b16a Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 18:40:21 +0200 Subject: [PATCH 081/173] chore: fix tests --- router/pkg/pubsub/nats/engine_datasource_factory_test.go | 5 ++++- router/pkg/pubsub/nats/engine_datasource_test.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/router/pkg/pubsub/nats/engine_datasource_factory_test.go b/router/pkg/pubsub/nats/engine_datasource_factory_test.go index 8eb28d473f..808a87c26d 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" + "go.uber.org/zap" ) func TestNatsEngineDataSourceFactory(t *testing.T) { @@ -168,6 +169,8 @@ func TestNatsEngineDataSourceFactoryWithStreamConfiguration(t *testing.T) { func TestEngineDataSourceFactory_RequestDataSource(t *testing.T) { // Create mock adapter mockAdapter := NewMockAdapter(t) + pubSubProvider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) + provider := datasource.NewHookedProvider(pubSubProvider, []datasource.OnStreamEventsFn{}, []datasource.OnPublishEventsFn{}) // Configure mock expectations for Request mockAdapter.On("Request", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { @@ -185,7 +188,7 @@ func TestEngineDataSourceFactory_RequestDataSource(t *testing.T) { eventType: EventTypeRequest, subjects: []string{"test-subject"}, fieldName: "testField", - NatsAdapter: mockAdapter, + NatsAdapter: provider, } // Get the data source diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 2c6609b87e..b9323bd19a 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { @@ -296,10 +297,12 @@ func TestNatsRequestDataSource_Load(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewMockAdapter(t) + pubSubProvider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) + provider := datasource.NewHookedProvider(pubSubProvider, []datasource.OnStreamEventsFn{}, []datasource.OnPublishEventsFn{}) tt.mockSetup(mockAdapter) dataSource := &NatsRequestDataSource{ - pubSub: mockAdapter, + pubSub: provider, } ctx := context.Background() From 036538be4bd4a2853829212ef57fb7c61c755b86 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 20:08:15 +0200 Subject: [PATCH 082/173] chore: removed useless adapters --- router/pkg/pubsub/nats/provider_builder.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/router/pkg/pubsub/nats/provider_builder.go b/router/pkg/pubsub/nats/provider_builder.go index d3e791bd74..91eed2e839 100644 --- a/router/pkg/pubsub/nats/provider_builder.go +++ b/router/pkg/pubsub/nats/provider_builder.go @@ -20,7 +20,6 @@ type ProviderBuilder struct { logger *zap.Logger hostName string routerListenAddr string - adapters map[string]Adapter } func (p *ProviderBuilder) TypeID() string { @@ -65,15 +64,10 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventCon } func (p *ProviderBuilder) BuildProvider(provider config.NatsEventSource) (datasource.Provider, error) { - adapterBase, pubSubProvider, err := buildProvider(p.ctx, provider, p.logger, p.hostName, p.routerListenAddr) + pubSubProvider, err := buildProvider(p.ctx, provider, p.logger, p.hostName, p.routerListenAddr) if err != nil { return nil, err } - adapter, ok := adapterBase.(Adapter) - if !ok { - return nil, fmt.Errorf("adapter for provider %s is not an Adapter", provider.ID) - } - p.adapters[provider.ID] = adapter return pubSubProvider, nil } @@ -122,18 +116,18 @@ func buildNatsOptions(eventSource config.NatsEventSource, logger *zap.Logger) ([ return opts, nil } -func buildProvider(ctx context.Context, provider config.NatsEventSource, logger *zap.Logger, hostName string, routerListenAddr string) (datasource.ProviderBase, datasource.Provider, error) { +func buildProvider(ctx context.Context, provider config.NatsEventSource, logger *zap.Logger, hostName string, routerListenAddr string) (datasource.Provider, error) { options, err := buildNatsOptions(provider, logger) if err != nil { - return nil, nil, fmt.Errorf("failed to build options for Nats provider with ID \"%s\": %w", provider.ID, err) + return nil, fmt.Errorf("failed to build options for Nats provider with ID \"%s\": %w", provider.ID, err) } adapter, err := NewAdapter(ctx, logger, provider.URL, options, hostName, routerListenAddr) if err != nil { - return nil, nil, fmt.Errorf("failed to create adapter for Nats provider with ID \"%s\": %w", provider.ID, err) + return nil, fmt.Errorf("failed to create adapter for Nats provider with ID \"%s\": %w", provider.ID, err) } pubSubProvider := datasource.NewPubSubProvider(provider.ID, providerTypeID, adapter, logger) - return adapter, pubSubProvider, nil + return pubSubProvider, nil } func NewProviderBuilder( @@ -147,6 +141,5 @@ func NewProviderBuilder( logger: logger, hostName: hostName, routerListenAddr: routerListenAddr, - adapters: make(map[string]Adapter), } } From 2c52b39507ebe4162bad5fb13a0239363a1cbf5c Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 20:11:51 +0200 Subject: [PATCH 083/173] chore: improved behaviour with multiple modules --- .../pkg/pubsub/datasource/hookedprovider.go | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/router/pkg/pubsub/datasource/hookedprovider.go b/router/pkg/pubsub/datasource/hookedprovider.go index a9f8186473..fa9b02e4f1 100644 --- a/router/pkg/pubsub/datasource/hookedprovider.go +++ b/router/pkg/pubsub/datasource/hookedprovider.go @@ -14,22 +14,19 @@ type hookedUpdater struct { } func (h *hookedUpdater) Update(events []StreamEvent) { - var newEvents []StreamEvent - var err error if len(h.OnStreamEventsFns) == 0 { h.updater.Update(events) return } - for _, fn := range h.OnStreamEventsFns { - newEvents, err = fn(h.ctx, h.subscriptionEventConfiguration, events) - if err != nil { - // TODO: do something with the error - continue - } + processedEvents, err := applyStreamEventHooks(h.ctx, h.subscriptionEventConfiguration, events, h.OnStreamEventsFns) + if err != nil { + // TODO: do something with the error - for now, continue with original events + h.updater.Update(events) + return } - h.updater.Update(newEvents) + h.updater.Update(processedEvents) } func (h *hookedUpdater) Complete() { @@ -40,6 +37,34 @@ func (h *hookedUpdater) Close(kind resolve.SubscriptionCloseKind) { h.updater.Close(kind) } +// applyStreamEventHooks processes events through a chain of hook functions +// Each hook receives the result from the previous hook, creating a proper middleware pipeline +func applyStreamEventHooks(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent, hooks []OnStreamEventsFn) ([]StreamEvent, error) { + currentEvents := events + for _, hook := range hooks { + var err error + currentEvents, err = hook(ctx, cfg, currentEvents) + if err != nil { + return nil, err + } + } + return currentEvents, nil +} + +// applyPublishEventHooks processes events through a chain of hook functions +// Each hook receives the result from the previous hook, creating a proper middleware pipeline +func applyPublishEventHooks(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent, hooks []OnPublishEventsFn) ([]StreamEvent, error) { + currentEvents := events + for _, hook := range hooks { + var err error + currentEvents, err = hook(ctx, cfg, currentEvents) + if err != nil { + return nil, err + } + } + return currentEvents, nil +} + func NewHookedProvider(provider Provider, onStreamEventsFns []OnStreamEventsFn, onPublishEventsFns []OnPublishEventsFn) Provider { return &HookedProvider{ OnStreamEventsFns: onStreamEventsFns, @@ -66,20 +91,16 @@ func (h *HookedProvider) Subscribe(ctx context.Context, cfg SubscriptionEventCon } func (h *HookedProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { - var newEvents []StreamEvent - var err error if len(h.OnPublishEventsFns) == 0 { return h.Provider.Publish(ctx, cfg, events) } - for _, fn := range h.OnPublishEventsFns { - newEvents, err = fn(ctx, cfg, events) - if err != nil { - return err - } + processedEvents, err := applyPublishEventHooks(ctx, cfg, events, h.OnPublishEventsFns) + if err != nil { + return err } - return h.Provider.Publish(ctx, cfg, newEvents) + return h.Provider.Publish(ctx, cfg, processedEvents) } func (h *HookedProvider) ID() string { From f9be6a13fe427f34aeebba4941f1507600b93596 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 28 Jul 2025 21:00:09 +0200 Subject: [PATCH 084/173] chore: separate private from public data --- router/pkg/pubsub/kafka/engine_datasource.go | 60 ++++++++++++------- .../pubsub/kafka/engine_datasource_factory.go | 2 +- .../pubsub/kafka/engine_datasource_test.go | 15 +++-- router/pkg/pubsub/nats/engine_datasource.go | 47 ++++++++++----- .../pubsub/nats/engine_datasource_factory.go | 4 +- .../pkg/pubsub/nats/engine_datasource_test.go | 14 ++--- router/pkg/pubsub/redis/engine_datasource.go | 32 +++++++--- .../pubsub/redis/engine_datasource_factory.go | 4 +- .../pubsub/redis/engine_datasource_test.go | 9 ++- 9 files changed, 117 insertions(+), 70 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index c203a76ac8..92babaa15a 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -25,6 +25,8 @@ func (e *Event) GetData() []byte { return e.Data } +// SubscriptionEventConfiguration is a public type that is used to allow access to custom fields +// of the provider type SubscriptionEventConfiguration struct { Provider string `json:"providerId"` Topics []string `json:"topics"` @@ -46,10 +48,44 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { return s.FieldName } +// publishData is a private type that is used to pass data from the engine to the provider +type publishData struct { + Provider string `json:"providerId"` + Topic string `json:"topic"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` +} + +// PublishEventConfiguration returns the publish event configuration from the publishData type +func (p *publishData) PublishEventConfiguration() datasource.PublishEventConfiguration { + return &PublishEventConfiguration{ + Provider: p.Provider, + Topic: p.Topic, + FieldName: p.FieldName, + } +} + +func (p *publishData) MarshalJSONTemplate() (string, error) { + // The content of the data field could be not valid JSON, so we can't use json.Marshal + // e.g. {"id":$$0$$,"update":$$1$$} + headers := p.Event.Headers + if headers == nil { + headers = make(map[string][]byte) + } + + headersBytes, err := json.Marshal(headers) + if err != nil { + return "", err + } + + return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, p.Topic, p.Event.Data, p.Event.Key, headersBytes, p.Provider), nil +} + +// PublishEventConfiguration is a public type that is used to allow access to custom fields +// of the provider type PublishEventConfiguration struct { Provider string `json:"providerId"` Topic string `json:"topic"` - Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -68,22 +104,6 @@ func (p *PublishEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { - // The content of the data field could be not valid JSON, so we can't use json.Marshal - // e.g. {"id":$$0$$,"update":$$1$$} - headers := s.Event.Headers - if headers == nil { - headers = make(map[string][]byte) - } - - headersBytes, err := json.Marshal(headers) - if err != nil { - return "", err - } - - return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil -} - type SubscriptionDataSource struct { pubSub datasource.ProviderBase } @@ -136,13 +156,13 @@ type PublishDataSource struct { } func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { - var publishConfiguration PublishEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) + var publishData publishData + err := json.Unmarshal(input, &publishData) if err != nil { return err } - if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { + if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index 535a4ebb60..9201eaed65 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -48,7 +48,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return "", fmt.Errorf("publish events should define one topic but received %d", len(c.topics)) } - evtCfg := PublishEventConfiguration{ + evtCfg := publishData{ Provider: c.providerId, Topic: c.topics[0], Event: Event{Data: eventData}, diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 0067859e18..148b0ab0aa 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -16,15 +16,15 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { +func TestPublishData_MarshalJSONTemplate(t *testing.T) { tests := []struct { name string - config PublishEventConfiguration + config publishData wantPattern string }{ { name: "simple configuration", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider", Topic: "test-topic", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, @@ -33,7 +33,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { }, { name: "with special characters", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, @@ -42,7 +42,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { }, { name: "with key", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Key: []byte("blablabla"), Data: json.RawMessage(`{}`)}, @@ -51,7 +51,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { }, { name: "with headers", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Headers: map[string][]byte{"key": []byte(`blablabla`)}, Data: json.RawMessage(`{}`)}, @@ -202,8 +202,7 @@ func TestKafkaPublishDataSource_Load(t *testing.T) { mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && - event.Topic == "test-topic" && - string(event.Event.Data) == `{"message":"hello"}` + event.Topic == "test-topic" }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 57a92923aa..851eeed9d7 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -52,10 +52,31 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { return s.FieldName } +// publishData is a private type that is used to pass data from the engine to the provider +type publishData struct { + Provider string `json:"providerId"` + Subject string `json:"subject"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` +} + +func (p *publishData) PublishEventConfiguration() datasource.PublishEventConfiguration { + return &PublishAndRequestEventConfiguration{ + Provider: p.Provider, + Subject: p.Subject, + FieldName: p.FieldName, + } +} + +func (p *publishData) MarshalJSONTemplate() (string, error) { + // The content of the data field could be not valid JSON, so we can't use json.Marshal + // e.g. {"id":$$0$$,"update":$$1$$} + return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, p.Subject, p.Event.Data, p.Provider), nil +} + type PublishAndRequestEventConfiguration struct { Provider string `json:"providerId"` Subject string `json:"subject"` - Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -74,12 +95,6 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() (string, error) { - // The content of the data field could be not valid JSON, so we can't use json.Marshal - // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()), nil -} - type SubscriptionSource struct { pubSub datasource.ProviderBase } @@ -133,13 +148,13 @@ type NatsPublishDataSource struct { } func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { - var publishConfiguration PublishAndRequestEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) + var publishData publishData + err := json.Unmarshal(input, &publishData) if err != nil { return err } - if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { + if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } @@ -156,28 +171,28 @@ type NatsRequestDataSource struct { } func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { - var subscriptionConfiguration PublishAndRequestEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) + var publishData publishData + err := json.Unmarshal(input, &publishData) if err != nil { return err } hookedProvider, ok := s.pubSub.(*datasource.HookedProvider) if !ok { - return fmt.Errorf("adapter for provider %s is not of the right hooked type", subscriptionConfiguration.ProviderID()) + return fmt.Errorf("adapter for provider %s is not of the right hooked type", publishData.Provider) } providerBase, ok := hookedProvider.Provider.(*datasource.PubSubProvider) if !ok { - return fmt.Errorf("adapter for provider %s is not of the right type", subscriptionConfiguration.ProviderID()) + return fmt.Errorf("adapter for provider %s is not of the right type", publishData.Provider) } adapter, ok := providerBase.Adapter.(Adapter) if !ok { - return fmt.Errorf("adapter for provider %s is not of the right type", subscriptionConfiguration.ProviderID()) + return fmt.Errorf("adapter for provider %s is not of the right type", publishData.Provider) } - return adapter.Request(ctx, &subscriptionConfiguration, &subscriptionConfiguration.Event, out) + return adapter.Request(ctx, publishData.PublishEventConfiguration(), &publishData.Event, out) } func (s *NatsRequestDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) error { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 68dea2a8e6..7538c1ae9b 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -61,11 +61,11 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri subject := c.subjects[0] - evtCfg := PublishAndRequestEventConfiguration{ + evtCfg := publishData{ Provider: c.providerId, Subject: subject, - Event: Event{Data: eventData}, FieldName: c.fieldName, + Event: Event{Data: eventData}, } return evtCfg.MarshalJSONTemplate() diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index b9323bd19a..98ee0445fb 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -21,12 +21,12 @@ import ( func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { tests := []struct { name string - config PublishAndRequestEventConfiguration + config publishData wantPattern string }{ { name: "simple configuration", - config: PublishAndRequestEventConfiguration{ + config: publishData{ Provider: "test-provider", Subject: "test-subject", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, @@ -35,7 +35,7 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { }, { name: "with special characters", - config: PublishAndRequestEventConfiguration{ + config: publishData{ Provider: "test-provider-id", Subject: "subject-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, @@ -48,7 +48,7 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { result, err := tt.config.MarshalJSONTemplate() assert.NoError(t, err) - assert.Equal(t, tt.wantPattern, result) + assert.Equal(t, tt.wantPattern, string(result)) }) } } @@ -186,8 +186,7 @@ func TestNatsPublishDataSource_Load(t *testing.T) { mockSetup: func(m *MockAdapter) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && - event.Subject == "test-subject" && - string(event.Event.Data) == `{"message":"hello"}` + event.Subject == "test-subject" }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) @@ -263,8 +262,7 @@ func TestNatsRequestDataSource_Load(t *testing.T) { mockSetup: func(m *MockAdapter) { m.On("Request", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { return event.ProviderID() == "test-provider" && - event.Subject == "test-subject" && - string(event.Event.Data) == `{"message":"hello"}` + event.Subject == "test-subject" }), mock.MatchedBy(func(event datasource.StreamEvent) bool { return event != nil && strings.EqualFold(string(event.GetData()), `{"message":"hello"}`) }), mock.Anything).Run(func(args mock.Arguments) { diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index a46f41c6be..eb63340fff 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -45,11 +45,31 @@ func (s *SubscriptionEventConfiguration) RootFieldName() string { return s.FieldName } +// publishData is a private type that is used to pass data from the engine to the provider + +type publishData struct { + Provider string `json:"providerId"` + Channel string `json:"channel"` + Event Event `json:"event"` + FieldName string `json:"rootFieldName"` +} + +func (p *publishData) PublishEventConfiguration() datasource.PublishEventConfiguration { + return &PublishEventConfiguration{ + Provider: p.Provider, + Channel: p.Channel, + FieldName: p.FieldName, + } +} + +func (p *publishData) MarshalJSONTemplate() (string, error) { + return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s"}`, p.Channel, p.Event.Data, p.Provider), nil +} + // PublishEventConfiguration contains configuration for publish events type PublishEventConfiguration struct { Provider string `json:"providerId"` Channel string `json:"channel"` - Event Event `json:"event"` // this should be in a different and private type, only used internally FieldName string `json:"rootFieldName"` } @@ -68,10 +88,6 @@ func (p *PublishEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { - return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID()), nil -} - // SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis type SubscriptionDataSource struct { pubSub Adapter @@ -134,13 +150,13 @@ type PublishDataSource struct { // Load processes a request to publish to Redis func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { - var publishConfiguration PublishEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) + var publishData publishData + err := json.Unmarshal(input, &publishData) if err != nil { return err } - if err := s.pubSub.Publish(ctx, &publishConfiguration, []datasource.StreamEvent{&publishConfiguration.Event}); err != nil { + if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { _, err = io.WriteString(out, `{"success": false}`) return err } diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index 0db9b5a8f4..d49e5bbd0f 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -58,11 +58,11 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri channel := channels[0] providerId := c.providerId - evtCfg := PublishEventConfiguration{ + evtCfg := publishData{ Provider: providerId, Channel: channel, - Event: Event{Data: eventData}, FieldName: c.fieldName, + Event: Event{Data: eventData}, } return evtCfg.MarshalJSONTemplate() diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 8c4859d36c..e0e8236511 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -19,12 +19,12 @@ import ( func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { tests := []struct { name string - config PublishEventConfiguration + config publishData wantPattern string }{ { name: "simple configuration", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider", Channel: "test-channel", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, @@ -33,7 +33,7 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { }, { name: "with special characters", - config: PublishEventConfiguration{ + config: publishData{ Provider: "test-provider-id", Channel: "channel-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, @@ -184,8 +184,7 @@ func TestRedisPublishDataSource_Load(t *testing.T) { mockSetup: func(m *datasource.MockProvider) { m.On("Publish", mock.Anything, mock.MatchedBy(func(event *PublishEventConfiguration) bool { return event.ProviderID() == "test-provider" && - event.Channel == "test-channel" && - string(event.Event.Data) == `{"message":"hello"}` + event.Channel == "test-channel" }), mock.MatchedBy(func(events []datasource.StreamEvent) bool { return len(events) == 1 && strings.EqualFold(string(events[0].GetData()), `{"message":"hello"}`) })).Return(nil) From 9f661f3510891f240520afcc07a9bbb17c3118bc Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 29 Jul 2025 14:56:31 +0200 Subject: [PATCH 085/173] chore: update graphql-go-tools to v2.0.0-rc.213 and use new SubscriptionConfiguration field name --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/factoryresolver.go | 16 +++++++++++++++- router/go.mod | 2 +- router/go.sum | 4 ++-- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 011455bae6..d47f42cb0d 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 516e3c2e53..36afa53a20 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 6827834db4..c520bb10d3 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -419,6 +419,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod for i, fn := range l.subscriptionHooks.onStart { subscriptionOnStartFns[i] = NewEngineSubscriptionOnStartHook(fn) } + customConfiguration, err := graphql_datasource.NewConfiguration(graphql_datasource.ConfigurationInput{ Fetch: &graphql_datasource.FetchConfiguration{ URL: fetchUrl, @@ -432,7 +433,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - SubscriptionOnStartFns: subscriptionOnStartFns, + StartupHooks: subscriptionOnStartFns, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, @@ -478,6 +479,17 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod for i, fn := range l.subscriptionHooks.onStart { subscriptionOnStartFns[i] = NewPubSubSubscriptionOnStartHook(fn) } + + onPublishEventsFns := make([]pubsub_datasource.OnPublishEventsFn, len(l.subscriptionHooks.onPublishEvents)) + for i, fn := range l.subscriptionHooks.onPublishEvents { + onPublishEventsFns[i] = NewPubSubOnPublishEventsHook(fn) + } + + onStreamEventsFns := make([]pubsub_datasource.OnStreamEventsFn, len(l.subscriptionHooks.onStreamEvents)) + for i, fn := range l.subscriptionHooks.onStreamEvents { + onStreamEventsFns[i] = NewPubSubOnStreamEventsHook(fn) + } + factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, routerEngineConfig.Events, @@ -487,6 +499,8 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod l.resolver.InstanceData().ListenAddress, pubsub.Hooks{ SubscriptionOnStart: subscriptionOnStartFns, + OnStreamEvents: onStreamEventsFns, + OnPublishEvents: onPublishEventsFns, }, ) if err != nil { diff --git a/router/go.mod b/router/go.mod index a643fd45e3..cd2466aa53 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index fca1956b6b..16b0129b27 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From cb6d3267cae69c1dffd887e869022ec45f46421e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 29 Jul 2025 15:44:31 +0200 Subject: [PATCH 086/173] Revert "chore: update graphql-go-tools to v2.0.0-rc.213 and use new SubscriptionConfiguration field name" This reverts commit 9f661f3510891f240520afcc07a9bbb17c3118bc. --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/factoryresolver.go | 16 +--------------- router/go.mod | 2 +- router/go.sum | 4 ++-- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index d47f42cb0d..011455bae6 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 36afa53a20..516e3c2e53 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index c520bb10d3..6827834db4 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -419,7 +419,6 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod for i, fn := range l.subscriptionHooks.onStart { subscriptionOnStartFns[i] = NewEngineSubscriptionOnStartHook(fn) } - customConfiguration, err := graphql_datasource.NewConfiguration(graphql_datasource.ConfigurationInput{ Fetch: &graphql_datasource.FetchConfiguration{ URL: fetchUrl, @@ -433,7 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - StartupHooks: subscriptionOnStartFns, + SubscriptionOnStartFns: subscriptionOnStartFns, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, @@ -479,17 +478,6 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod for i, fn := range l.subscriptionHooks.onStart { subscriptionOnStartFns[i] = NewPubSubSubscriptionOnStartHook(fn) } - - onPublishEventsFns := make([]pubsub_datasource.OnPublishEventsFn, len(l.subscriptionHooks.onPublishEvents)) - for i, fn := range l.subscriptionHooks.onPublishEvents { - onPublishEventsFns[i] = NewPubSubOnPublishEventsHook(fn) - } - - onStreamEventsFns := make([]pubsub_datasource.OnStreamEventsFn, len(l.subscriptionHooks.onStreamEvents)) - for i, fn := range l.subscriptionHooks.onStreamEvents { - onStreamEventsFns[i] = NewPubSubOnStreamEventsHook(fn) - } - factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( l.ctx, routerEngineConfig.Events, @@ -499,8 +487,6 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod l.resolver.InstanceData().ListenAddress, pubsub.Hooks{ SubscriptionOnStart: subscriptionOnStartFns, - OnStreamEvents: onStreamEventsFns, - OnPublishEvents: onPublishEventsFns, }, ) if err != nil { diff --git a/router/go.mod b/router/go.mod index cd2466aa53..a643fd45e3 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 16b0129b27..fca1956b6b 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From bff87177c42133a34e818f11d0fd3ae82aafd1e9 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 29 Jul 2025 15:47:18 +0200 Subject: [PATCH 087/173] chore: update graphql-go-tools to v2.0.0-rc.213 and use new SubscriptionConfiguration field name --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/core/factoryresolver.go | 2 +- router/go.mod | 2 +- router/go.sum | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 011455bae6..d47f42cb0d 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 516e3c2e53..36afa53a20 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 6827834db4..25cf7d84c7 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -432,7 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - SubscriptionOnStartFns: subscriptionOnStartFns, + StartupHooks: subscriptionOnStartFns, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, diff --git a/router/go.mod b/router/go.mod index a643fd45e3..cd2466aa53 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index fca1956b6b..16b0129b27 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987 h1:cPWOgyh+WcF+rDGgfuDGodPy0YH+MkSQdTuCp0XofTE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.210.0.20250724163305-7bb743179987/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 21c63666a96d60f4ad8555a0cc000d6d09dc1007 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 29 Jul 2025 15:50:33 +0200 Subject: [PATCH 088/173] chore: improve adr --- adr/cosmo-streams-v1.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index d5929b8538..44fddf0408 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -17,7 +17,8 @@ This ADR describes new hooks that will be added to the router to support more cu The goal is to allow developers to customize the cosmo streams behavior. ## Decision -A developer can implement a custom module by creating a struct that implements the following interfaces: +The following interfaces will extend the existing logic in the custom modules. +These provide additional control over subscriptions by providing hooks, which are invoked during specific events. - `SubscriptionOnStartHandler`: Called once at subscription start. - `StreamBatchEventHook`: Called each time a batch of events is received from the provider. From 5a11bb647f5c746a9beadd795ecd5c9e3c82eafd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 29 Jul 2025 19:06:55 +0200 Subject: [PATCH 089/173] chore: remove hookedprovider and embedded hooks logics inside pubsubprovider --- router/core/factoryresolver.go | 2 +- .../pkg/pubsub/datasource/hookedprovider.go | 120 ------------------ router/pkg/pubsub/datasource/mocks.go | 80 ++++++++++++ router/pkg/pubsub/datasource/provider.go | 4 + .../pkg/pubsub/datasource/pubsubprovider.go | 98 +++++++++++++- router/pkg/pubsub/nats/engine_datasource.go | 7 +- .../nats/engine_datasource_factory_test.go | 3 +- .../pkg/pubsub/nats/engine_datasource_test.go | 3 +- router/pkg/pubsub/pubsub.go | 8 +- router/pkg/pubsub/pubsub_test.go | 4 + 10 files changed, 187 insertions(+), 142 deletions(-) delete mode 100644 router/pkg/pubsub/datasource/hookedprovider.go diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 59a85b6047..4d93616bf8 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -432,7 +432,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod ForwardedClientHeaderNames: forwardedClientHeaders, ForwardedClientHeaderRegularExpressions: forwardedClientRegexps, WsSubProtocol: wsSubprotocol, - SubscriptionOnStartFns: subscriptionOnStartFns, + StartupHooks: subscriptionOnStartFns, }, SchemaConfiguration: schemaConfiguration, CustomScalarTypeFields: customScalarTypeFields, diff --git a/router/pkg/pubsub/datasource/hookedprovider.go b/router/pkg/pubsub/datasource/hookedprovider.go deleted file mode 100644 index fa9b02e4f1..0000000000 --- a/router/pkg/pubsub/datasource/hookedprovider.go +++ /dev/null @@ -1,120 +0,0 @@ -package datasource - -import ( - "context" - - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" -) - -type hookedUpdater struct { - ctx context.Context - updater SubscriptionEventUpdater - subscriptionEventConfiguration SubscriptionEventConfiguration - OnStreamEventsFns []OnStreamEventsFn -} - -func (h *hookedUpdater) Update(events []StreamEvent) { - if len(h.OnStreamEventsFns) == 0 { - h.updater.Update(events) - return - } - - processedEvents, err := applyStreamEventHooks(h.ctx, h.subscriptionEventConfiguration, events, h.OnStreamEventsFns) - if err != nil { - // TODO: do something with the error - for now, continue with original events - h.updater.Update(events) - return - } - - h.updater.Update(processedEvents) -} - -func (h *hookedUpdater) Complete() { - h.updater.Complete() -} - -func (h *hookedUpdater) Close(kind resolve.SubscriptionCloseKind) { - h.updater.Close(kind) -} - -// applyStreamEventHooks processes events through a chain of hook functions -// Each hook receives the result from the previous hook, creating a proper middleware pipeline -func applyStreamEventHooks(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent, hooks []OnStreamEventsFn) ([]StreamEvent, error) { - currentEvents := events - for _, hook := range hooks { - var err error - currentEvents, err = hook(ctx, cfg, currentEvents) - if err != nil { - return nil, err - } - } - return currentEvents, nil -} - -// applyPublishEventHooks processes events through a chain of hook functions -// Each hook receives the result from the previous hook, creating a proper middleware pipeline -func applyPublishEventHooks(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent, hooks []OnPublishEventsFn) ([]StreamEvent, error) { - currentEvents := events - for _, hook := range hooks { - var err error - currentEvents, err = hook(ctx, cfg, currentEvents) - if err != nil { - return nil, err - } - } - return currentEvents, nil -} - -func NewHookedProvider(provider Provider, onStreamEventsFns []OnStreamEventsFn, onPublishEventsFns []OnPublishEventsFn) Provider { - return &HookedProvider{ - OnStreamEventsFns: onStreamEventsFns, - OnPublishEventsFns: onPublishEventsFns, - Provider: provider, - } -} - -type HookedProvider struct { - Provider - OnPublishEventsFns []OnPublishEventsFn - OnStreamEventsFns []OnStreamEventsFn -} - -func (h *HookedProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { - hookedUpdater := &hookedUpdater{ - ctx: ctx, - updater: updater, - subscriptionEventConfiguration: cfg, - OnStreamEventsFns: h.OnStreamEventsFns, - } - - return h.Provider.Subscribe(ctx, cfg, hookedUpdater) -} - -func (h *HookedProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { - if len(h.OnPublishEventsFns) == 0 { - return h.Provider.Publish(ctx, cfg, events) - } - - processedEvents, err := applyPublishEventHooks(ctx, cfg, events, h.OnPublishEventsFns) - if err != nil { - return err - } - - return h.Provider.Publish(ctx, cfg, processedEvents) -} - -func (h *HookedProvider) ID() string { - return h.Provider.ID() -} - -func (h *HookedProvider) TypeID() string { - return h.Provider.TypeID() -} - -func (h *HookedProvider) Startup(ctx context.Context) error { - return h.Provider.Startup(ctx) -} - -func (h *HookedProvider) Shutdown(ctx context.Context) error { - return h.Provider.Shutdown(ctx) -} diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index c6b1153429..bdd75e1cd7 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -619,6 +619,86 @@ func (_c *MockProvider_Publish_Call) RunAndReturn(run func(ctx context.Context, return _c } +// SetOnPublishEventsFns provides a mock function for the type MockProvider +func (_mock *MockProvider) SetOnPublishEventsFns(onPublishEventsFns []OnPublishEventsFn) { + _mock.Called(onPublishEventsFns) + return +} + +// MockProvider_SetOnPublishEventsFns_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetOnPublishEventsFns' +type MockProvider_SetOnPublishEventsFns_Call struct { + *mock.Call +} + +// SetOnPublishEventsFns is a helper method to define mock.On call +// - onPublishEventsFns []OnPublishEventsFn +func (_e *MockProvider_Expecter) SetOnPublishEventsFns(onPublishEventsFns interface{}) *MockProvider_SetOnPublishEventsFns_Call { + return &MockProvider_SetOnPublishEventsFns_Call{Call: _e.mock.On("SetOnPublishEventsFns", onPublishEventsFns)} +} + +func (_c *MockProvider_SetOnPublishEventsFns_Call) Run(run func(onPublishEventsFns []OnPublishEventsFn)) *MockProvider_SetOnPublishEventsFns_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 []OnPublishEventsFn + if args[0] != nil { + arg0 = args[0].([]OnPublishEventsFn) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockProvider_SetOnPublishEventsFns_Call) Return() *MockProvider_SetOnPublishEventsFns_Call { + _c.Call.Return() + return _c +} + +func (_c *MockProvider_SetOnPublishEventsFns_Call) RunAndReturn(run func(onPublishEventsFns []OnPublishEventsFn)) *MockProvider_SetOnPublishEventsFns_Call { + _c.Run(run) + return _c +} + +// SetOnStreamEventsFns provides a mock function for the type MockProvider +func (_mock *MockProvider) SetOnStreamEventsFns(onStreamEventsFns []OnStreamEventsFn) { + _mock.Called(onStreamEventsFns) + return +} + +// MockProvider_SetOnStreamEventsFns_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetOnStreamEventsFns' +type MockProvider_SetOnStreamEventsFns_Call struct { + *mock.Call +} + +// SetOnStreamEventsFns is a helper method to define mock.On call +// - onStreamEventsFns []OnStreamEventsFn +func (_e *MockProvider_Expecter) SetOnStreamEventsFns(onStreamEventsFns interface{}) *MockProvider_SetOnStreamEventsFns_Call { + return &MockProvider_SetOnStreamEventsFns_Call{Call: _e.mock.On("SetOnStreamEventsFns", onStreamEventsFns)} +} + +func (_c *MockProvider_SetOnStreamEventsFns_Call) Run(run func(onStreamEventsFns []OnStreamEventsFn)) *MockProvider_SetOnStreamEventsFns_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 []OnStreamEventsFn + if args[0] != nil { + arg0 = args[0].([]OnStreamEventsFn) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockProvider_SetOnStreamEventsFns_Call) Return() *MockProvider_SetOnStreamEventsFns_Call { + _c.Call.Return() + return _c +} + +func (_c *MockProvider_SetOnStreamEventsFns_Call) RunAndReturn(run func(onStreamEventsFns []OnStreamEventsFn)) *MockProvider_SetOnStreamEventsFns_Call { + _c.Run(run) + return _c +} + // Shutdown provides a mock function for the type MockProvider func (_mock *MockProvider) Shutdown(ctx context.Context) error { ret := _mock.Called(ctx) diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index e103be53b1..030ef20516 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -32,6 +32,10 @@ type Provider interface { ID() string // TypeID Get the provider type id (e.g. "kafka", "nats") TypeID() string + // SetOnPublishEventsFns Set the functions that will be called before publishing events + SetOnPublishEventsFns([]OnPublishEventsFn) + // SetOnStreamEventsFns Set the functions that will be called when receiving events + SetOnStreamEventsFns([]OnStreamEventsFn) } // ProviderBuilder is the interface that the provider builder must implement. diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index ccbec577fa..6eba7fbf4b 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -3,14 +3,76 @@ package datasource import ( "context" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) type PubSubProvider struct { - id string - typeID string - Adapter ProviderBase - Logger *zap.Logger + onPublishEventsFns []OnPublishEventsFn + onStreamEventsFns []OnStreamEventsFn + id string + typeID string + Adapter ProviderBase + Logger *zap.Logger +} + +type hookedUpdater struct { + ctx context.Context + updater SubscriptionEventUpdater + subscriptionEventConfiguration SubscriptionEventConfiguration + OnStreamEventsFns []OnStreamEventsFn +} + +func (h *hookedUpdater) Update(events []StreamEvent) { + if len(h.OnStreamEventsFns) == 0 { + h.updater.Update(events) + return + } + + processedEvents, err := applyStreamEventHooks(h.ctx, h.subscriptionEventConfiguration, events, h.OnStreamEventsFns) + if err != nil { + // TODO: do something with the error - for now, continue with original events + h.updater.Update(events) + return + } + + h.updater.Update(processedEvents) +} + +func (h *hookedUpdater) Complete() { + h.updater.Complete() +} + +func (h *hookedUpdater) Close(kind resolve.SubscriptionCloseKind) { + h.updater.Close(kind) +} + +// applyStreamEventHooks processes events through a chain of hook functions +// Each hook receives the result from the previous hook, creating a proper middleware pipeline +func applyStreamEventHooks(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent, hooks []OnStreamEventsFn) ([]StreamEvent, error) { + currentEvents := events + for _, hook := range hooks { + var err error + currentEvents, err = hook(ctx, cfg, currentEvents) + if err != nil { + return nil, err + } + } + return currentEvents, nil +} + +// applyPublishEventHooks processes events through a chain of hook functions +// Each hook receives the result from the previous hook, creating a proper middleware pipeline +func applyPublishEventHooks(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent, hooks []OnPublishEventsFn) ([]StreamEvent, error) { + currentEvents := events + for _, hook := range hooks { + var err error + currentEvents, err = hook(ctx, cfg, currentEvents) + if err != nil { + return nil, err + } + } + return currentEvents, nil } func (p *PubSubProvider) ID() string { @@ -36,11 +98,35 @@ func (p *PubSubProvider) Shutdown(ctx context.Context) error { } func (p *PubSubProvider) Subscribe(ctx context.Context, conf SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { - return p.Adapter.Subscribe(ctx, conf, updater) + hookedUpdater := &hookedUpdater{ + ctx: ctx, + updater: updater, + subscriptionEventConfiguration: conf, + OnStreamEventsFns: p.onStreamEventsFns, + } + + return p.Adapter.Subscribe(ctx, conf, hookedUpdater) } func (p *PubSubProvider) Publish(ctx context.Context, conf PublishEventConfiguration, events []StreamEvent) error { - return p.Adapter.Publish(ctx, conf, events) + if len(p.onPublishEventsFns) == 0 { + return p.Adapter.Publish(ctx, conf, events) + } + + processedEvents, err := applyPublishEventHooks(ctx, conf, events, p.onPublishEventsFns) + if err != nil { + return err + } + + return p.Adapter.Publish(ctx, conf, processedEvents) +} + +func (p *PubSubProvider) SetOnPublishEventsFns(fns []OnPublishEventsFn) { + p.onPublishEventsFns = fns +} + +func (p *PubSubProvider) SetOnStreamEventsFns(fns []OnStreamEventsFn) { + p.onStreamEventsFns = fns } func NewPubSubProvider(id string, typeID string, adapter ProviderBase, logger *zap.Logger) *PubSubProvider { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 851eeed9d7..cabea836bd 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -177,12 +177,7 @@ func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *byt return err } - hookedProvider, ok := s.pubSub.(*datasource.HookedProvider) - if !ok { - return fmt.Errorf("adapter for provider %s is not of the right hooked type", publishData.Provider) - } - - providerBase, ok := hookedProvider.Provider.(*datasource.PubSubProvider) + providerBase, ok := s.pubSub.(*datasource.PubSubProvider) if !ok { return fmt.Errorf("adapter for provider %s is not of the right type", publishData.Provider) } diff --git a/router/pkg/pubsub/nats/engine_datasource_factory_test.go b/router/pkg/pubsub/nats/engine_datasource_factory_test.go index 808a87c26d..6de24569f9 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory_test.go @@ -169,8 +169,7 @@ func TestNatsEngineDataSourceFactoryWithStreamConfiguration(t *testing.T) { func TestEngineDataSourceFactory_RequestDataSource(t *testing.T) { // Create mock adapter mockAdapter := NewMockAdapter(t) - pubSubProvider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) - provider := datasource.NewHookedProvider(pubSubProvider, []datasource.OnStreamEventsFn{}, []datasource.OnPublishEventsFn{}) + provider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) // Configure mock expectations for Request mockAdapter.On("Request", mock.Anything, mock.MatchedBy(func(event *PublishAndRequestEventConfiguration) bool { diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 98ee0445fb..30f733b869 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -295,8 +295,7 @@ func TestNatsRequestDataSource_Load(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewMockAdapter(t) - pubSubProvider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) - provider := datasource.NewHookedProvider(pubSubProvider, []datasource.OnStreamEventsFn{}, []datasource.OnPublishEventsFn{}) + provider := datasource.NewPubSubProvider("test-provider", "nats", mockAdapter, zap.NewNop()) tt.mockSetup(mockAdapter) dataSource := &NatsRequestDataSource{ diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index 21405a470b..589ed34075 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -149,11 +149,9 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder if err != nil { return nil, nil, err } - pubSubProviders[provider.ID()] = pubsub_datasource.NewHookedProvider( - provider, - hooks.OnStreamEvents, - hooks.OnPublishEvents, - ) + provider.SetOnStreamEventsFns(hooks.OnStreamEvents) + provider.SetOnPublishEventsFns(hooks.OnPublishEvents) + pubSubProviders[provider.ID()] = provider } // check if all used providers are initialized diff --git a/router/pkg/pubsub/pubsub_test.go b/router/pkg/pubsub/pubsub_test.go index 8085fdd24e..d35d8bdbef 100644 --- a/router/pkg/pubsub/pubsub_test.go +++ b/router/pkg/pubsub/pubsub_test.go @@ -59,6 +59,8 @@ func TestBuild_OK(t *testing.T) { } mockPubSubProvider.On("ID").Return("provider-1") + mockPubSubProvider.On("SetOnStreamEventsFns", []datasource.OnStreamEventsFn(nil)).Return(nil) + mockPubSubProvider.On("SetOnPublishEventsFns", []datasource.OnPublishEventsFn(nil)).Return(nil) mockBuilder.On("TypeID").Return("nats") mockBuilder.On("BuildProvider", natsEventSources[0]).Return(mockPubSubProvider, nil) @@ -234,6 +236,8 @@ func TestBuild_ShouldNotInitializeProviderIfNotUsed(t *testing.T) { } mockPubSubUsedProvider.On("ID").Return("provider-2") + mockPubSubUsedProvider.On("SetOnStreamEventsFns", []datasource.OnStreamEventsFn(nil)).Return(nil) + mockPubSubUsedProvider.On("SetOnPublishEventsFns", []datasource.OnPublishEventsFn(nil)).Return(nil) mockBuilder.On("TypeID").Return("nats") mockBuilder.On("BuildProvider", natsEventSources[1]).Return(mockPubSubUsedProvider, nil) From 7f6d7915806cbbdb53bb59fc2ae17faf690146ba Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 08:53:04 +0200 Subject: [PATCH 090/173] chore: better names --- .../availability/subgraph/schema.resolvers.go | 2 - .../mood/subgraph/schema.resolvers.go | 2 - router/.mockery.yml | 2 +- router/pkg/pubsub/datasource/mocks.go | 54 +++++++++---------- router/pkg/pubsub/datasource/provider.go | 14 ++--- .../pkg/pubsub/datasource/pubsubprovider.go | 4 +- router/pkg/pubsub/kafka/adapter.go | 4 +- router/pkg/pubsub/kafka/engine_datasource.go | 4 +- .../pubsub/kafka/engine_datasource_factory.go | 2 +- router/pkg/pubsub/nats/adapter.go | 4 +- router/pkg/pubsub/nats/engine_datasource.go | 6 +-- .../pubsub/nats/engine_datasource_factory.go | 2 +- router/pkg/pubsub/redis/adapter.go | 2 +- 13 files changed, 49 insertions(+), 53 deletions(-) diff --git a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go index c768f446d7..10f14593de 100644 --- a/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/availability/subgraph/schema.resolvers.go @@ -19,7 +19,6 @@ func (r *mutationResolver) UpdateAvailability(ctx context.Context, employeeID in storage.Set(employeeID, isAvailable) conf := &nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdated.%d", employeeID)), - Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, } evt := &nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))} @@ -30,7 +29,6 @@ func (r *mutationResolver) UpdateAvailability(ctx context.Context, employeeID in conf2 := &nats.PublishAndRequestEventConfiguration{ Subject: r.GetPubSubName(fmt.Sprintf("employeeUpdatedMyNats.%d", employeeID)), - Event: nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))}, } evt2 := &nats.Event{Data: []byte(fmt.Sprintf(`{"id":%d,"__typename": "Employee"}`, employeeID))} err = r.NatsPubSubByProviderID["my-nats"].Publish(ctx, conf2, []datasource.StreamEvent{evt2}) diff --git a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go index 6417798b16..b9b426593c 100644 --- a/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go +++ b/demo/pkg/subgraphs/mood/subgraph/schema.resolvers.go @@ -22,7 +22,6 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood if r.NatsPubSubByProviderID["default"] != nil { err := r.NatsPubSubByProviderID["default"].Publish(ctx, &nats.PublishAndRequestEventConfiguration{ Subject: myNatsTopic, - Event: nats.Event{Data: []byte(payload)}, }, []datasource.StreamEvent{&nats.Event{Data: []byte(payload)}}) if err != nil { return nil, err @@ -35,7 +34,6 @@ func (r *mutationResolver) UpdateMood(ctx context.Context, employeeID int, mood if r.NatsPubSubByProviderID["my-nats"] != nil { err := r.NatsPubSubByProviderID["my-nats"].Publish(ctx, &nats.PublishAndRequestEventConfiguration{ Subject: defaultTopic, - Event: nats.Event{Data: []byte(payload)}, }, []datasource.StreamEvent{&nats.Event{Data: []byte(payload)}}) if err != nil { return nil, err diff --git a/router/.mockery.yml b/router/.mockery.yml index b84fa2f4fa..436ed0eb14 100644 --- a/router/.mockery.yml +++ b/router/.mockery.yml @@ -13,7 +13,7 @@ template-schema: '{{.Template}}.schema.json' packages: github.com/wundergraph/cosmo/router/pkg/pubsub/datasource: interfaces: - ProviderLifecycle: + Lifecycle: ProviderBuilder: EngineDataSourceFactory: Provider: diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index bdd75e1cd7..b15e3eed4d 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -356,13 +356,13 @@ func (_c *MockEngineDataSourceFactory_TransformEventData_Call) RunAndReturn(run return _c } -// NewMockProviderLifecycle creates a new instance of MockProviderLifecycle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// NewMockLifecycle creates a new instance of MockLifecycle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewMockProviderLifecycle(t interface { +func NewMockLifecycle(t interface { mock.TestingT Cleanup(func()) -}) *MockProviderLifecycle { - mock := &MockProviderLifecycle{} +}) *MockLifecycle { + mock := &MockLifecycle{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) @@ -370,21 +370,21 @@ func NewMockProviderLifecycle(t interface { return mock } -// MockProviderLifecycle is an autogenerated mock type for the ProviderLifecycle type -type MockProviderLifecycle struct { +// MockLifecycle is an autogenerated mock type for the Lifecycle type +type MockLifecycle struct { mock.Mock } -type MockProviderLifecycle_Expecter struct { +type MockLifecycle_Expecter struct { mock *mock.Mock } -func (_m *MockProviderLifecycle) EXPECT() *MockProviderLifecycle_Expecter { - return &MockProviderLifecycle_Expecter{mock: &_m.Mock} +func (_m *MockLifecycle) EXPECT() *MockLifecycle_Expecter { + return &MockLifecycle_Expecter{mock: &_m.Mock} } -// Shutdown provides a mock function for the type MockProviderLifecycle -func (_mock *MockProviderLifecycle) Shutdown(ctx context.Context) error { +// Shutdown provides a mock function for the type MockLifecycle +func (_mock *MockLifecycle) Shutdown(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { @@ -400,18 +400,18 @@ func (_mock *MockProviderLifecycle) Shutdown(ctx context.Context) error { return r0 } -// MockProviderLifecycle_Shutdown_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Shutdown' -type MockProviderLifecycle_Shutdown_Call struct { +// MockLifecycle_Shutdown_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Shutdown' +type MockLifecycle_Shutdown_Call struct { *mock.Call } // Shutdown is a helper method to define mock.On call // - ctx context.Context -func (_e *MockProviderLifecycle_Expecter) Shutdown(ctx interface{}) *MockProviderLifecycle_Shutdown_Call { - return &MockProviderLifecycle_Shutdown_Call{Call: _e.mock.On("Shutdown", ctx)} +func (_e *MockLifecycle_Expecter) Shutdown(ctx interface{}) *MockLifecycle_Shutdown_Call { + return &MockLifecycle_Shutdown_Call{Call: _e.mock.On("Shutdown", ctx)} } -func (_c *MockProviderLifecycle_Shutdown_Call) Run(run func(ctx context.Context)) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) Run(run func(ctx context.Context)) *MockLifecycle_Shutdown_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -424,18 +424,18 @@ func (_c *MockProviderLifecycle_Shutdown_Call) Run(run func(ctx context.Context) return _c } -func (_c *MockProviderLifecycle_Shutdown_Call) Return(err error) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) Return(err error) *MockLifecycle_Shutdown_Call { _c.Call.Return(err) return _c } -func (_c *MockProviderLifecycle_Shutdown_Call) RunAndReturn(run func(ctx context.Context) error) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) RunAndReturn(run func(ctx context.Context) error) *MockLifecycle_Shutdown_Call { _c.Call.Return(run) return _c } -// Startup provides a mock function for the type MockProviderLifecycle -func (_mock *MockProviderLifecycle) Startup(ctx context.Context) error { +// Startup provides a mock function for the type MockLifecycle +func (_mock *MockLifecycle) Startup(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { @@ -451,18 +451,18 @@ func (_mock *MockProviderLifecycle) Startup(ctx context.Context) error { return r0 } -// MockProviderLifecycle_Startup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Startup' -type MockProviderLifecycle_Startup_Call struct { +// MockLifecycle_Startup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Startup' +type MockLifecycle_Startup_Call struct { *mock.Call } // Startup is a helper method to define mock.On call // - ctx context.Context -func (_e *MockProviderLifecycle_Expecter) Startup(ctx interface{}) *MockProviderLifecycle_Startup_Call { - return &MockProviderLifecycle_Startup_Call{Call: _e.mock.On("Startup", ctx)} +func (_e *MockLifecycle_Expecter) Startup(ctx interface{}) *MockLifecycle_Startup_Call { + return &MockLifecycle_Startup_Call{Call: _e.mock.On("Startup", ctx)} } -func (_c *MockProviderLifecycle_Startup_Call) Run(run func(ctx context.Context)) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) Run(run func(ctx context.Context)) *MockLifecycle_Startup_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -475,12 +475,12 @@ func (_c *MockProviderLifecycle_Startup_Call) Run(run func(ctx context.Context)) return _c } -func (_c *MockProviderLifecycle_Startup_Call) Return(err error) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) Return(err error) *MockLifecycle_Startup_Call { _c.Call.Return(err) return _c } -func (_c *MockProviderLifecycle_Startup_Call) RunAndReturn(run func(ctx context.Context) error) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) RunAndReturn(run func(ctx context.Context) error) *MockLifecycle_Startup_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 030ef20516..b6ae2f5282 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -8,26 +8,26 @@ import ( type ArgumentTemplateCallback func(tpl string) (string, error) -// ProviderLifecycle is the interface that the provider must implement +// Lifecycle is the interface that the provider must implement // to allow the router to start and stop the provider -type ProviderLifecycle interface { +type Lifecycle interface { // Startup is the method called when the provider is started Startup(ctx context.Context) error // Shutdown is the method called when the provider is shut down Shutdown(ctx context.Context) error } -// ProviderBase is the interface that the provider must implement -// to implement the base functionality -type ProviderBase interface { - ProviderLifecycle +// Adapter is the interface that the provider must implement +// to implement the basic functionality +type Adapter interface { + Lifecycle Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error } // Provider is the interface that the PubSub provider must implement type Provider interface { - ProviderBase + Adapter // ID Get the provider ID as specified in the configuration ID() string // TypeID Get the provider type id (e.g. "kafka", "nats") diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 6eba7fbf4b..4bb8e0accd 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -12,7 +12,7 @@ type PubSubProvider struct { onStreamEventsFns []OnStreamEventsFn id string typeID string - Adapter ProviderBase + Adapter Adapter Logger *zap.Logger } @@ -129,7 +129,7 @@ func (p *PubSubProvider) SetOnStreamEventsFns(fns []OnStreamEventsFn) { p.onStreamEventsFns = fns } -func NewPubSubProvider(id string, typeID string, adapter ProviderBase, logger *zap.Logger) *PubSubProvider { +func NewPubSubProvider(id string, typeID string, adapter Adapter, logger *zap.Logger) *PubSubProvider { return &PubSubProvider{ id: id, typeID: typeID, diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 0d6258d00a..4d62fa8371 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -18,8 +18,8 @@ var ( errClientClosed = errors.New("client closed") ) -// Ensure ProviderAdapter implements ProviderBase -var _ datasource.ProviderBase = (*ProviderAdapter)(nil) +// Ensure ProviderAdapter implements Adapter +var _ datasource.Adapter = (*ProviderAdapter)(nil) // ProviderAdapter is a Kafka pubsub implementation. // It uses the franz-go Kafka client to consume and produce messages. diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 92babaa15a..c6ebf5675c 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -105,7 +105,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { } type SubscriptionDataSource struct { - pubSub datasource.ProviderBase + pubSub datasource.Adapter } func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { @@ -152,7 +152,7 @@ func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updat } type PublishDataSource struct { - pubSub datasource.ProviderBase + pubSub datasource.Adapter } func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index 9201eaed65..0a0061af6b 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -21,7 +21,7 @@ type EngineDataSourceFactory struct { topics []string providerId string - KafkaAdapter datasource.ProviderBase + KafkaAdapter datasource.Adapter } func (c *EngineDataSourceFactory) GetFieldName() string { diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 2310eb2469..a4f69d4039 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -17,13 +17,13 @@ import ( // Adapter defines the methods that a NATS adapter should implement type Adapter interface { - datasource.ProviderBase + datasource.Adapter // Request sends a request to the specified subject and writes the response to the given writer Request(ctx context.Context, cfg datasource.PublishEventConfiguration, event datasource.StreamEvent, w io.Writer) error } // Ensure ProviderAdapter implements ProviderSubscriptionHooks -var _ datasource.ProviderBase = (*ProviderAdapter)(nil) +var _ datasource.Adapter = (*ProviderAdapter)(nil) // ProviderAdapter implements the AdapterInterface for NATS pub/sub type ProviderAdapter struct { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index cabea836bd..c0df4d8d38 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -96,7 +96,7 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { } type SubscriptionSource struct { - pubSub datasource.ProviderBase + pubSub datasource.Adapter } func (s *SubscriptionSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { @@ -144,7 +144,7 @@ func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater d } type NatsPublishDataSource struct { - pubSub datasource.ProviderBase + pubSub datasource.Adapter } func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { @@ -167,7 +167,7 @@ func (s *NatsPublishDataSource) LoadWithFiles(ctx context.Context, input []byte, } type NatsRequestDataSource struct { - pubSub datasource.ProviderBase + pubSub datasource.Adapter } func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 7538c1ae9b..622e9382c4 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -18,7 +18,7 @@ const ( ) type EngineDataSourceFactory struct { - NatsAdapter datasource.ProviderBase + NatsAdapter datasource.Adapter fieldName string eventType EventType diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index a3e302fba2..0c844b4d93 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -23,7 +23,7 @@ type Adapter interface { } // Ensure ProviderAdapter implements ProviderSubscriptionHooks -var _ datasource.ProviderBase = (*ProviderAdapter)(nil) +var _ datasource.Adapter = (*ProviderAdapter)(nil) func NewProviderAdapter(ctx context.Context, logger *zap.Logger, urls []string, clusterEnabled bool) Adapter { ctx, cancel := context.WithCancel(ctx) From 35aceb035099ce27dc5c334c5e0ee3bf57b2aae8 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 13:19:24 +0200 Subject: [PATCH 091/173] refactor: rename ProviderLifecycle to Lifecycle and replaced hookeddatasource with a common subscription data source --- router/.mockery.yml | 2 +- router/pkg/pubsub/datasource/datasource.go | 7 +- .../pkg/pubsub/datasource/hookeddatasource.go | 30 ---- router/pkg/pubsub/datasource/mocks.go | 133 +++++++++++++----- router/pkg/pubsub/datasource/planner.go | 8 +- router/pkg/pubsub/datasource/provider.go | 15 +- .../pkg/pubsub/datasource/pubsubprovider.go | 8 +- .../pubsub/datasource/pubsubprovider_test.go | 8 +- .../datasource/subscription_datasource.go | 66 +++++++++ router/pkg/pubsub/kafka/adapter.go | 16 ++- router/pkg/pubsub/kafka/engine_datasource.go | 50 ------- .../pubsub/kafka/engine_datasource_factory.go | 28 +++- .../kafka/engine_datasource_factory_test.go | 58 ++++++++ .../pubsub/kafka/engine_datasource_test.go | 121 ---------------- router/pkg/pubsub/kafka/mocks.go | 14 +- router/pkg/pubsub/nats/adapter.go | 30 ++-- router/pkg/pubsub/nats/engine_datasource.go | 55 +------- .../pubsub/nats/engine_datasource_factory.go | 28 +++- .../nats/engine_datasource_factory_test.go | 57 ++++++++ .../pkg/pubsub/nats/engine_datasource_test.go | 121 ---------------- router/pkg/pubsub/nats/mocks.go | 14 +- router/pkg/pubsub/redis/adapter.go | 18 ++- router/pkg/pubsub/redis/engine_datasource.go | 58 -------- .../pubsub/redis/engine_datasource_factory.go | 28 +++- .../redis/engine_datasource_factory_test.go | 58 ++++++++ .../pubsub/redis/engine_datasource_test.go | 121 ---------------- router/pkg/pubsub/redis/mocks.go | 14 +- router/pkg/pubsub/redis/provider_builder.go | 2 +- 28 files changed, 500 insertions(+), 668 deletions(-) delete mode 100644 router/pkg/pubsub/datasource/hookeddatasource.go create mode 100644 router/pkg/pubsub/datasource/subscription_datasource.go diff --git a/router/.mockery.yml b/router/.mockery.yml index 8ea750cc0e..558bca2185 100644 --- a/router/.mockery.yml +++ b/router/.mockery.yml @@ -13,7 +13,7 @@ template-schema: '{{.Template}}.schema.json' packages: github.com/wundergraph/cosmo/router/pkg/pubsub/datasource: interfaces: - ProviderLifecycle: + Lifecycle: ProviderBuilder: EngineDataSourceFactory: Provider: diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index c61909f495..b93add32e1 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -5,10 +5,11 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -type PubSubSubscriptionDataSource interface { +type SubscriptionDataSource interface { SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration - Start(ctx *resolve.Context, input []byte, updater SubscriptionEventUpdater) error + Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) + SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) } // EngineDataSourceFactory is the interface that all pubsub data sources must implement. @@ -30,7 +31,7 @@ type EngineDataSourceFactory interface { // ResolveDataSourceSubscription returns the engine SubscriptionDataSource implementation // that contains methods to start a subscription, which will be called by the Planner // when a subscription is initiated - ResolveDataSourceSubscription() (PubSubSubscriptionDataSource, error) + ResolveDataSourceSubscription() (SubscriptionDataSource, error) // ResolveDataSourceSubscriptionInput build the input that will be passed to the engine SubscriptionDataSource ResolveDataSourceSubscriptionInput() (string, error) // TransformEventData allows the data source to transform the event data using the extractFn diff --git a/router/pkg/pubsub/datasource/hookeddatasource.go b/router/pkg/pubsub/datasource/hookeddatasource.go deleted file mode 100644 index 6e6bed217b..0000000000 --- a/router/pkg/pubsub/datasource/hookeddatasource.go +++ /dev/null @@ -1,30 +0,0 @@ -package datasource - -import ( - "github.com/cespare/xxhash/v2" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" -) - -type HookedSubscriptionDataSource struct { - SubscriptionOnStartFns []SubscriptionOnStartFn - SubscriptionDataSource PubSubSubscriptionDataSource -} - -func (h *HookedSubscriptionDataSource) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { - for _, fn := range h.SubscriptionOnStartFns { - close, err = fn(ctx, h.SubscriptionDataSource.SubscriptionEventConfiguration(input)) - if err != nil || close { - return - } - } - - return -} - -func (h *HookedSubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - return h.SubscriptionDataSource.Start(ctx, input, NewSubscriptionEventUpdater(updater)) -} - -func (h *HookedSubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) { - return h.SubscriptionDataSource.UniqueRequestID(ctx, input, xxh) -} diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index 1968b4bba1..68b847714d 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -198,23 +198,23 @@ func (_c *MockEngineDataSourceFactory_ResolveDataSourceInput_Call) RunAndReturn( } // ResolveDataSourceSubscription provides a mock function for the type MockEngineDataSourceFactory -func (_mock *MockEngineDataSourceFactory) ResolveDataSourceSubscription() (PubSubSubscriptionDataSource, error) { +func (_mock *MockEngineDataSourceFactory) ResolveDataSourceSubscription() (SubscriptionDataSource, error) { ret := _mock.Called() if len(ret) == 0 { panic("no return value specified for ResolveDataSourceSubscription") } - var r0 PubSubSubscriptionDataSource + var r0 SubscriptionDataSource var r1 error - if returnFunc, ok := ret.Get(0).(func() (PubSubSubscriptionDataSource, error)); ok { + if returnFunc, ok := ret.Get(0).(func() (SubscriptionDataSource, error)); ok { return returnFunc() } - if returnFunc, ok := ret.Get(0).(func() PubSubSubscriptionDataSource); ok { + if returnFunc, ok := ret.Get(0).(func() SubscriptionDataSource); ok { r0 = returnFunc() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(PubSubSubscriptionDataSource) + r0 = ret.Get(0).(SubscriptionDataSource) } } if returnFunc, ok := ret.Get(1).(func() error); ok { @@ -242,12 +242,12 @@ func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Run(ru return _c } -func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Return(pubSubSubscriptionDataSource PubSubSubscriptionDataSource, err error) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { - _c.Call.Return(pubSubSubscriptionDataSource, err) +func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) Return(subscriptionDataSource SubscriptionDataSource, err error) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { + _c.Call.Return(subscriptionDataSource, err) return _c } -func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) RunAndReturn(run func() (PubSubSubscriptionDataSource, error)) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { +func (_c *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call) RunAndReturn(run func() (SubscriptionDataSource, error)) *MockEngineDataSourceFactory_ResolveDataSourceSubscription_Call { _c.Call.Return(run) return _c } @@ -356,13 +356,13 @@ func (_c *MockEngineDataSourceFactory_TransformEventData_Call) RunAndReturn(run return _c } -// NewMockProviderLifecycle creates a new instance of MockProviderLifecycle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// NewMockLifecycle creates a new instance of MockLifecycle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewMockProviderLifecycle(t interface { +func NewMockLifecycle(t interface { mock.TestingT Cleanup(func()) -}) *MockProviderLifecycle { - mock := &MockProviderLifecycle{} +}) *MockLifecycle { + mock := &MockLifecycle{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) @@ -370,21 +370,21 @@ func NewMockProviderLifecycle(t interface { return mock } -// MockProviderLifecycle is an autogenerated mock type for the ProviderLifecycle type -type MockProviderLifecycle struct { +// MockLifecycle is an autogenerated mock type for the Lifecycle type +type MockLifecycle struct { mock.Mock } -type MockProviderLifecycle_Expecter struct { +type MockLifecycle_Expecter struct { mock *mock.Mock } -func (_m *MockProviderLifecycle) EXPECT() *MockProviderLifecycle_Expecter { - return &MockProviderLifecycle_Expecter{mock: &_m.Mock} +func (_m *MockLifecycle) EXPECT() *MockLifecycle_Expecter { + return &MockLifecycle_Expecter{mock: &_m.Mock} } -// Shutdown provides a mock function for the type MockProviderLifecycle -func (_mock *MockProviderLifecycle) Shutdown(ctx context.Context) error { +// Shutdown provides a mock function for the type MockLifecycle +func (_mock *MockLifecycle) Shutdown(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { @@ -400,18 +400,18 @@ func (_mock *MockProviderLifecycle) Shutdown(ctx context.Context) error { return r0 } -// MockProviderLifecycle_Shutdown_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Shutdown' -type MockProviderLifecycle_Shutdown_Call struct { +// MockLifecycle_Shutdown_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Shutdown' +type MockLifecycle_Shutdown_Call struct { *mock.Call } // Shutdown is a helper method to define mock.On call // - ctx context.Context -func (_e *MockProviderLifecycle_Expecter) Shutdown(ctx interface{}) *MockProviderLifecycle_Shutdown_Call { - return &MockProviderLifecycle_Shutdown_Call{Call: _e.mock.On("Shutdown", ctx)} +func (_e *MockLifecycle_Expecter) Shutdown(ctx interface{}) *MockLifecycle_Shutdown_Call { + return &MockLifecycle_Shutdown_Call{Call: _e.mock.On("Shutdown", ctx)} } -func (_c *MockProviderLifecycle_Shutdown_Call) Run(run func(ctx context.Context)) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) Run(run func(ctx context.Context)) *MockLifecycle_Shutdown_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -424,18 +424,18 @@ func (_c *MockProviderLifecycle_Shutdown_Call) Run(run func(ctx context.Context) return _c } -func (_c *MockProviderLifecycle_Shutdown_Call) Return(err error) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) Return(err error) *MockLifecycle_Shutdown_Call { _c.Call.Return(err) return _c } -func (_c *MockProviderLifecycle_Shutdown_Call) RunAndReturn(run func(ctx context.Context) error) *MockProviderLifecycle_Shutdown_Call { +func (_c *MockLifecycle_Shutdown_Call) RunAndReturn(run func(ctx context.Context) error) *MockLifecycle_Shutdown_Call { _c.Call.Return(run) return _c } -// Startup provides a mock function for the type MockProviderLifecycle -func (_mock *MockProviderLifecycle) Startup(ctx context.Context) error { +// Startup provides a mock function for the type MockLifecycle +func (_mock *MockLifecycle) Startup(ctx context.Context) error { ret := _mock.Called(ctx) if len(ret) == 0 { @@ -451,18 +451,18 @@ func (_mock *MockProviderLifecycle) Startup(ctx context.Context) error { return r0 } -// MockProviderLifecycle_Startup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Startup' -type MockProviderLifecycle_Startup_Call struct { +// MockLifecycle_Startup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Startup' +type MockLifecycle_Startup_Call struct { *mock.Call } // Startup is a helper method to define mock.On call // - ctx context.Context -func (_e *MockProviderLifecycle_Expecter) Startup(ctx interface{}) *MockProviderLifecycle_Startup_Call { - return &MockProviderLifecycle_Startup_Call{Call: _e.mock.On("Startup", ctx)} +func (_e *MockLifecycle_Expecter) Startup(ctx interface{}) *MockLifecycle_Startup_Call { + return &MockLifecycle_Startup_Call{Call: _e.mock.On("Startup", ctx)} } -func (_c *MockProviderLifecycle_Startup_Call) Run(run func(ctx context.Context)) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) Run(run func(ctx context.Context)) *MockLifecycle_Startup_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -475,12 +475,12 @@ func (_c *MockProviderLifecycle_Startup_Call) Run(run func(ctx context.Context)) return _c } -func (_c *MockProviderLifecycle_Startup_Call) Return(err error) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) Return(err error) *MockLifecycle_Startup_Call { _c.Call.Return(err) return _c } -func (_c *MockProviderLifecycle_Startup_Call) RunAndReturn(run func(ctx context.Context) error) *MockProviderLifecycle_Startup_Call { +func (_c *MockLifecycle_Startup_Call) RunAndReturn(run func(ctx context.Context) error) *MockLifecycle_Startup_Call { _c.Call.Return(run) return _c } @@ -658,6 +658,69 @@ func (_c *MockProvider_Startup_Call) RunAndReturn(run func(ctx context.Context) return _c } +// Subscribe provides a mock function for the type MockProvider +func (_mock *MockProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + ret := _mock.Called(ctx, cfg, updater) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, SubscriptionEventUpdater) error); ok { + r0 = returnFunc(ctx, cfg, updater) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockProvider_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe' +type MockProvider_Subscribe_Call struct { + *mock.Call +} + +// Subscribe is a helper method to define mock.On call +// - ctx context.Context +// - cfg SubscriptionEventConfiguration +// - updater SubscriptionEventUpdater +func (_e *MockProvider_Expecter) Subscribe(ctx interface{}, cfg interface{}, updater interface{}) *MockProvider_Subscribe_Call { + return &MockProvider_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, cfg, updater)} +} + +func (_c *MockProvider_Subscribe_Call) Run(run func(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater)) *MockProvider_Subscribe_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 SubscriptionEventConfiguration + if args[1] != nil { + arg1 = args[1].(SubscriptionEventConfiguration) + } + var arg2 SubscriptionEventUpdater + if args[2] != nil { + arg2 = args[2].(SubscriptionEventUpdater) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockProvider_Subscribe_Call) Return(err error) *MockProvider_Subscribe_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockProvider_Subscribe_Call) RunAndReturn(run func(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error) *MockProvider_Subscribe_Call { + _c.Call.Return(run) + return _c +} + // TypeID provides a mock function for the type MockProvider func (_mock *MockProvider) TypeID() string { ret := _mock.Called() diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index c9e3b098b6..b60a647d6e 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -116,11 +116,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to get resolve data source subscription: %w", err)) return plan.SubscriptionConfiguration{} } - - hookedDataSource := &HookedSubscriptionDataSource{ - SubscriptionDataSource: dataSource, - SubscriptionOnStartFns: p.config.SubscriptionOnStartFns, - } + dataSource.SetSubscriptionOnStartFns(p.config.SubscriptionOnStartFns...) input, err := pubSubDataSource.ResolveDataSourceSubscriptionInput() if err != nil { @@ -131,7 +127,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati return plan.SubscriptionConfiguration{ Input: input, Variables: p.variables, - DataSource: hookedDataSource, + DataSource: dataSource, PostProcessing: resolve.PostProcessingConfiguration{ MergePath: []string{pubSubDataSource.GetFieldName()}, }, diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index c72c816b0d..0a699e2b14 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -8,16 +8,25 @@ import ( type ArgumentTemplateCallback func(tpl string) (string, error) -type ProviderLifecycle interface { +// Lifecycle is the interface that the provider must implement +// to allow the router to start and stop the provider +type Lifecycle interface { // Startup is the method called when the provider is started Startup(ctx context.Context) error // Shutdown is the method called when the provider is shut down Shutdown(ctx context.Context) error } +// Adapter is the interface that the provider must implement +// to implement the basic functionality +type Adapter interface { + Lifecycle + Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error +} + // Provider is the interface that the PubSub provider must implement type Provider interface { - ProviderLifecycle + Adapter // ID Get the provider ID as specified in the configuration ID() string // TypeID Get the provider type id (e.g. "kafka", "nats") @@ -45,7 +54,7 @@ const ( // StreamEvent is a generic interface for all stream events // Each provider will have its own event type that implements this interface -// there could be common fields in future, but for now we don't need any +// there could be other common fields in the future, but for now we only have data type StreamEvent interface { GetData() []byte } diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 9e1223d950..84561b06db 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -9,7 +9,7 @@ import ( type PubSubProvider struct { id string typeID string - Adapter ProviderLifecycle + Adapter Adapter Logger *zap.Logger } @@ -35,7 +35,11 @@ func (p *PubSubProvider) Shutdown(ctx context.Context) error { return nil } -func NewPubSubProvider(id string, typeID string, adapter ProviderLifecycle, logger *zap.Logger) *PubSubProvider { +func (p *PubSubProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + return p.Adapter.Subscribe(ctx, cfg, updater) +} + +func NewPubSubProvider(id string, typeID string, adapter Adapter, logger *zap.Logger) *PubSubProvider { return &PubSubProvider{ id: id, typeID: typeID, diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index 6579b62072..134bfbd6bb 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -10,7 +10,7 @@ import ( ) func TestProvider_Startup_Success(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Startup", mock.Anything).Return(nil) provider := PubSubProvider{ @@ -22,7 +22,7 @@ func TestProvider_Startup_Success(t *testing.T) { } func TestProvider_Startup_Error(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Startup", mock.Anything).Return(errors.New("connect error")) provider := PubSubProvider{ @@ -34,7 +34,7 @@ func TestProvider_Startup_Error(t *testing.T) { } func TestProvider_Shutdown_Success(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Shutdown", mock.Anything).Return(nil) provider := PubSubProvider{ @@ -46,7 +46,7 @@ func TestProvider_Shutdown_Success(t *testing.T) { } func TestProvider_Shutdown_Error(t *testing.T) { - mockAdapter := NewMockProviderLifecycle(t) + mockAdapter := NewMockProvider(t) mockAdapter.On("Shutdown", mock.Anything).Return(errors.New("close error")) provider := PubSubProvider{ diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go new file mode 100644 index 0000000000..709d43b2ed --- /dev/null +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -0,0 +1,66 @@ +package datasource + +import ( + "encoding/json" + "fmt" + + "github.com/cespare/xxhash/v2" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +type uniqueRequestIdFn func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error + +type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { + pubSub Adapter + uniqueRequestID uniqueRequestIdFn + subscriptionOnStartFns []SubscriptionOnStartFn +} + +func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration { + var subscriptionConfiguration C + err := json.Unmarshal(input, &subscriptionConfiguration) + if err != nil { + return nil + } + return subscriptionConfiguration +} + +func (s *PubSubSubscriptionDataSource[C]) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return s.uniqueRequestID(ctx, input, xxh) +} + +func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { + subConf := s.SubscriptionEventConfiguration(input) + if subConf == nil { + return fmt.Errorf("no subscription configuration found") + } + + conf, ok := subConf.(C) + if !ok { + return fmt.Errorf("invalid subscription configuration") + } + + return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(updater)) +} + +func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { + for _, fn := range s.subscriptionOnStartFns { + close, err = fn(ctx, s.SubscriptionEventConfiguration(input)) + if err != nil || close { + return + } + } + + return +} + +func (s *PubSubSubscriptionDataSource[C]) SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) { + s.subscriptionOnStartFns = append(s.subscriptionOnStartFns, fns...) +} + +func NewPubSubSubscriptionDataSource[C SubscriptionEventConfiguration](pubSub Adapter, uniqueRequestIdFn uniqueRequestIdFn) *PubSubSubscriptionDataSource[C] { + return &PubSubSubscriptionDataSource[C]{ + pubSub: pubSub, + uniqueRequestID: uniqueRequestIdFn, + } +} diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 17b805f72e..b18b6f93fa 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -20,7 +20,7 @@ var ( // Adapter defines the interface for Kafka adapter operations type Adapter interface { - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error + Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error Publish(ctx context.Context, event PublishEventConfiguration) error Startup(ctx context.Context) error Shutdown(ctx context.Context) error @@ -104,23 +104,27 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u // Subscribe subscribes to the given topics and updates the subscription updater. // The engine already deduplicates subscriptions with the same topics, stream configuration, extensions, headers, etc. -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := conf.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", subConf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("topics", event.Topics), + zap.Strings("topics", subConf.Topics), ) // Create a new client for the topic client, err := kgo.NewClient(append(p.opts, - kgo.ConsumeTopics(event.Topics...), + kgo.ConsumeTopics(subConf.Topics...), // We want to consume the events produced after the first subscription was created // Messages are shared among all subscriptions, therefore old events are not redelivered // This replicates a stateless publish-subscribe model kgo.ConsumeResetOffset(kgo.NewOffset().AfterMilli(time.Now().UnixMilli())), // For observability, we set the client ID to "router" - kgo.ClientID(fmt.Sprintf("cosmo.router.consumer.%s", strings.Join(event.Topics, "-"))), + kgo.ClientID(fmt.Sprintf("cosmo.router.consumer.%s", strings.Join(subConf.Topics, "-"))), // FIXME: the client id should have some unique identifier, like in nats // What if we have multiple subscriptions for the same topics? // What if we have more router instances? diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 2b27e13363..d11b99b5d5 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -7,11 +7,8 @@ import ( "fmt" "io" - "github.com/buger/jsonparser" - "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) // Event represents an event from Kafka @@ -84,53 +81,6 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, s.Topic, s.Event.Data, s.Event.Key, headersBytes, s.ProviderID()), nil } -type SubscriptionDataSource struct { - pubSub Adapter -} - -func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return nil - } - return &subscriptionConfiguration -} - -func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { - val, _, _, err := jsonparser.Get(input, "topics") - if err != nil { - return err - } - - _, err = xxh.Write(val) - if err != nil { - return err - } - - val, _, _, err = jsonparser.Get(input, "providerId") - if err != nil { - return err - } - - _, err = xxh.Write(val) - return err -} - -func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { - subConf := s.SubscriptionEventConfiguration(input) - if subConf == nil { - return fmt.Errorf("no subscription configuration found") - } - - conf, ok := subConf.(*SubscriptionEventConfiguration) - if !ok { - return fmt.Errorf("invalid subscription configuration") - } - - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) -} - type PublishDataSource struct { pubSub Adapter } diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index b4672ebfb4..30507bc13b 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" + "github.com/buger/jsonparser" + "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) @@ -58,10 +60,28 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return evtCfg.MarshalJSONTemplate() } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { - return &SubscriptionDataSource{ - pubSub: c.KafkaAdapter, - }, nil +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSource, error) { + return datasource.NewPubSubSubscriptionDataSource[*SubscriptionEventConfiguration]( + c.KafkaAdapter, + func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + val, _, _, err := jsonparser.Get(input, "topics") + if err != nil { + return err + } + + _, err = xxh.Write(val) + if err != nil { + return err + } + + val, _, _, err = jsonparser.Get(input, "providerId") + if err != nil { + return err + } + + _, err = xxh.Write(val) + return err + }), nil } func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go index c1bd6f0d56..0b4ea9c59c 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory_test.go @@ -4,11 +4,15 @@ import ( "bytes" "context" "encoding/json" + "errors" "testing" + "github.com/cespare/xxhash/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestKafkaEngineDataSourceFactory(t *testing.T) { @@ -137,3 +141,57 @@ func TestKafkaEngineDataSourceFactoryMultiTopicSubscription(t *testing.T) { require.Equal(t, "test-topic-1", subscriptionConfig.Topics[0], "Expected first topic to be 'test-topic-1'") require.Equal(t, "test-topic-2", subscriptionConfig.Topics[1], "Expected second topic to be 'test-topic-2'") } + +func TestKafkaEngineDataSourceFactory_UniqueRequestID(t *testing.T) { + tests := []struct { + name string + input string + expectError bool + expectedError error + }{ + { + name: "valid input", + input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, + expectError: false, + }, + { + name: "missing topics", + input: `{"providerId":"test-provider"}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + { + name: "missing providerId", + input: `{"topics":["topic1", "topic2"]}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := &EngineDataSourceFactory{ + KafkaAdapter: NewMockAdapter(t), + } + source, err := factory.ResolveDataSourceSubscription() + require.NoError(t, err) + ctx := &resolve.Context{} + input := []byte(tt.input) + xxh := xxhash.New() + + err = source.UniqueRequestID(ctx, input, xxh) + + if tt.expectError { + require.Error(t, err) + if tt.expectedError != nil { + // For jsonparser errors, just check if the error message contains the expected text + assert.Contains(t, err.Error(), tt.expectedError.Error()) + } + } else { + require.NoError(t, err) + // Check that the hash has been updated + assert.NotEqual(t, 0, xxh.Sum64()) + } + }) + } +} diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 881b7779e0..eed485b246 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -7,12 +7,9 @@ import ( "errors" "testing" - "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { @@ -68,124 +65,6 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { } } -func TestSubscriptionSource_UniqueRequestID(t *testing.T) { - tests := []struct { - name string - input string - expectError bool - expectedError error - }{ - { - name: "valid input", - input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, - expectError: false, - }, - { - name: "missing topics", - input: `{"providerId":"test-provider"}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - { - name: "missing providerId", - input: `{"topics":["topic1", "topic2"]}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - source := &SubscriptionDataSource{ - pubSub: NewMockAdapter(t), - } - ctx := &resolve.Context{} - input := []byte(tt.input) - xxh := xxhash.New() - - err := source.UniqueRequestID(ctx, input, xxh) - - if tt.expectError { - require.Error(t, err) - if tt.expectedError != nil { - // For jsonparser errors, just check if the error message contains the expected text - assert.Contains(t, err.Error(), tt.expectedError.Error()) - } - } else { - require.NoError(t, err) - // Check that the hash has been updated - assert.NotEqual(t, 0, xxh.Sum64()) - } - }) - } -} - -func TestSubscriptionSource_Start(t *testing.T) { - tests := []struct { - name string - input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) - expectError bool - }{ - { - name: "successful subscription", - input: `{"topics":["topic1", "topic2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Topics: []string{"topic1", "topic2"}, - }, mock.Anything).Return(nil) - }, - expectError: false, - }, - { - name: "adapter returns error", - input: `{"topics":["topic1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Topics: []string{"topic1"}, - }, mock.Anything).Return(errors.New("subscription error")) - }, - expectError: true, - }, - { - name: "invalid input json", - input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, - expectError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionEventUpdater(t) - tt.mockSetup(mockAdapter, updater) - - source := &SubscriptionDataSource{ - pubSub: mockAdapter, - } - - // Set up go context - goCtx := context.Background() - - // Create a resolve.Context with the standard context - resolveCtx := &resolve.Context{} - resolveCtx = resolveCtx.WithContext(goCtx) - - input := []byte(tt.input) - err := source.Start(resolveCtx, input, updater) - - if tt.expectError { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - func TestKafkaPublishDataSource_Load(t *testing.T) { tests := []struct { name string diff --git a/router/pkg/pubsub/kafka/mocks.go b/router/pkg/pubsub/kafka/mocks.go index da945393bc..08faa08eb2 100644 --- a/router/pkg/pubsub/kafka/mocks.go +++ b/router/pkg/pubsub/kafka/mocks.go @@ -198,7 +198,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -206,7 +206,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -221,21 +221,21 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context -// - event SubscriptionEventConfiguration +// - event datasource.SubscriptionEventConfiguration // - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 SubscriptionEventConfiguration + var arg1 datasource.SubscriptionEventConfiguration if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) + arg1 = args[1].(datasource.SubscriptionEventConfiguration) } var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { @@ -255,7 +255,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 4e332c056f..f78577b898 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -18,7 +18,7 @@ import ( // Adapter defines the methods that a NATS adapter should implement type Adapter interface { // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error + Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error // Publish publishes the given event to the specified subject Publish(ctx context.Context, event PublishAndRequestEventConfiguration) error // Request sends a request to the specified subject and writes the response to the given writer @@ -71,11 +71,15 @@ func (p *ProviderAdapter) getDurableConsumerName(durableName string, subjects [] return fmt.Sprintf("%s-%x", durableName, subjHash.Sum64()), nil } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := conf.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", subConf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("subjects", event.Subjects), + zap.Strings("subjects", subConf.Subjects), ) if p.client == nil { @@ -86,24 +90,24 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent return datasource.NewError("nats jetstream not initialized", nil) } - if event.StreamConfiguration != nil { - durableConsumerName, err := p.getDurableConsumerName(event.StreamConfiguration.Consumer, event.Subjects) + if subConf.StreamConfiguration != nil { + durableConsumerName, err := p.getDurableConsumerName(subConf.StreamConfiguration.Consumer, subConf.Subjects) if err != nil { return err } consumerConfig := jetstream.ConsumerConfig{ Durable: durableConsumerName, - FilterSubjects: event.Subjects, + FilterSubjects: subConf.Subjects, } // Durable consumers are removed automatically only if the InactiveThreshold value is set - if event.StreamConfiguration.ConsumerInactiveThreshold > 0 { - consumerConfig.InactiveThreshold = time.Duration(event.StreamConfiguration.ConsumerInactiveThreshold) * time.Second + if subConf.StreamConfiguration.ConsumerInactiveThreshold > 0 { + consumerConfig.InactiveThreshold = time.Duration(subConf.StreamConfiguration.ConsumerInactiveThreshold) * time.Second } - consumer, err := p.js.CreateOrUpdateConsumer(ctx, event.StreamConfiguration.StreamName, consumerConfig) + consumer, err := p.js.CreateOrUpdateConsumer(ctx, subConf.StreamConfiguration.StreamName, consumerConfig) if err != nil { log.Error("creating or updating consumer", zap.Error(err)) - return datasource.NewError(fmt.Sprintf(`failed to create or update consumer for stream "%s"`, event.StreamConfiguration.StreamName), err) + return datasource.NewError(fmt.Sprintf(`failed to create or update consumer for stream "%s"`, subConf.StreamConfiguration.StreamName), err) } p.closeWg.Add(1) @@ -152,8 +156,8 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } msgChan := make(chan *nats.Msg) - subscriptions := make([]*nats.Subscription, len(event.Subjects)) - for i, subject := range event.Subjects { + subscriptions := make([]*nats.Subscription, len(subConf.Subjects)) + for i, subject := range subConf.Subjects { subscription, err := p.client.ChanSubscribe(subject, msgChan) if err != nil { log.Error("subscribing to NATS subject", zap.Error(err), zap.String("subscription_subject", subject)) diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index a00f30f4a4..5c12d361e3 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -7,11 +7,8 @@ import ( "fmt" "io" - "github.com/buger/jsonparser" - "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) // Event represents an event from NATS @@ -74,58 +71,10 @@ func (p *PublishAndRequestEventConfiguration) RootFieldName() string { return p.FieldName } -func (s *PublishAndRequestEventConfiguration) MarshalJSONTemplate() (string, error) { +func (p *PublishAndRequestEventConfiguration) MarshalJSONTemplate() (string, error) { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Subject, s.Event.Data, s.ProviderID()), nil -} - -type SubscriptionSource struct { - pubSub Adapter -} - -func (s *SubscriptionSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return nil - } - return &subscriptionConfiguration -} - -func (s *SubscriptionSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { - - val, _, _, err := jsonparser.Get(input, "subjects") - if err != nil { - return err - } - - _, err = xxh.Write(val) - if err != nil { - return err - } - - val, _, _, err = jsonparser.Get(input, "providerId") - if err != nil { - return err - } - - _, err = xxh.Write(val) - return err -} - -func (s *SubscriptionSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { - subConf := s.SubscriptionEventConfiguration(input) - if subConf == nil { - return fmt.Errorf("no subscription configuration found") - } - - conf, ok := subConf.(*SubscriptionEventConfiguration) - if !ok { - return fmt.Errorf("invalid subscription configuration") - } - - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) + return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, p.Subject, p.Event.Data, p.ProviderID()), nil } type NatsPublishDataSource struct { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index e43c49e8ea..36d3932e0d 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -5,6 +5,8 @@ import ( "fmt" "slices" + "github.com/buger/jsonparser" + "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" @@ -72,10 +74,28 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri return evtCfg.MarshalJSONTemplate() } -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { - return &SubscriptionSource{ - pubSub: c.NatsAdapter, - }, nil +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSource, error) { + return datasource.NewPubSubSubscriptionDataSource[*SubscriptionEventConfiguration]( + c.NatsAdapter, + func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + val, _, _, err := jsonparser.Get(input, "subjects") + if err != nil { + return err + } + + _, err = xxh.Write(val) + if err != nil { + return err + } + + val, _, _, err = jsonparser.Get(input, "providerId") + if err != nil { + return err + } + + _, err = xxh.Write(val) + return err + }), nil } func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { diff --git a/router/pkg/pubsub/nats/engine_datasource_factory_test.go b/router/pkg/pubsub/nats/engine_datasource_factory_test.go index 50e20a98e7..a94c8d5941 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory_test.go @@ -4,13 +4,16 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "testing" + "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestNatsEngineDataSourceFactory(t *testing.T) { @@ -253,3 +256,57 @@ func TestTransformEventConfig(t *testing.T) { assert.Contains(t, err.Error(), "invalid subject") }) } + +func TestNatsEngineDataSourceFactory_UniqueRequestID(t *testing.T) { + tests := []struct { + name string + input string + expectError bool + expectedError error + }{ + { + name: "valid input", + input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, + expectError: false, + }, + { + name: "missing subjects", + input: `{"providerId":"test-provider"}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + { + name: "missing providerId", + input: `{"subjects":["subject1", "subject2"]}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := &EngineDataSourceFactory{ + NatsAdapter: NewMockAdapter(t), + } + source, err := factory.ResolveDataSourceSubscription() + require.NoError(t, err) + ctx := &resolve.Context{} + input := []byte(tt.input) + xxh := xxhash.New() + + err = source.UniqueRequestID(ctx, input, xxh) + + if tt.expectError { + require.Error(t, err) + if tt.expectedError != nil { + // For jsonparser errors, just check if the error message contains the expected text + assert.Contains(t, err.Error(), tt.expectedError.Error()) + } + } else { + require.NoError(t, err) + // Check that the hash has been updated + assert.NotEqual(t, 0, xxh.Sum64()) + } + }) + } +} diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 79b8219e53..5d060d2c0d 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -8,12 +8,9 @@ import ( "io" "testing" - "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { @@ -51,124 +48,6 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { } } -func TestSubscriptionSource_UniqueRequestID(t *testing.T) { - tests := []struct { - name string - input string - expectError bool - expectedError error - }{ - { - name: "valid input", - input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, - expectError: false, - }, - { - name: "missing subjects", - input: `{"providerId":"test-provider"}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - { - name: "missing providerId", - input: `{"subjects":["subject1", "subject2"]}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - source := &SubscriptionSource{ - pubSub: NewMockAdapter(t), - } - ctx := &resolve.Context{} - input := []byte(tt.input) - xxh := xxhash.New() - - err := source.UniqueRequestID(ctx, input, xxh) - - if tt.expectError { - require.Error(t, err) - if tt.expectedError != nil { - // For jsonparser errors, just check if the error message contains the expected text - assert.Contains(t, err.Error(), tt.expectedError.Error()) - } - } else { - require.NoError(t, err) - // Check that the hash has been updated - assert.NotEqual(t, 0, xxh.Sum64()) - } - }) - } -} - -func TestSubscriptionSource_Start(t *testing.T) { - tests := []struct { - name string - input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) - expectError bool - }{ - { - name: "successful subscription", - input: `{"subjects":["subject1", "subject2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Subjects: []string{"subject1", "subject2"}, - }, mock.Anything).Return(nil) - }, - expectError: false, - }, - { - name: "adapter returns error", - input: `{"subjects":["subject1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Subjects: []string{"subject1"}, - }, mock.Anything).Return(errors.New("subscription error")) - }, - expectError: true, - }, - { - name: "invalid input json", - input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, - expectError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionEventUpdater(t) - tt.mockSetup(mockAdapter, updater) - - source := &SubscriptionSource{ - pubSub: mockAdapter, - } - - // Set up go context - goCtx := context.Background() - - // Create a resolve.Context with the standard context - resolveCtx := &resolve.Context{} - resolveCtx = resolveCtx.WithContext(goCtx) - - input := []byte(tt.input) - err := source.Start(resolveCtx, input, updater) - - if tt.expectError { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - func TestNatsPublishDataSource_Load(t *testing.T) { tests := []struct { name string diff --git a/router/pkg/pubsub/nats/mocks.go b/router/pkg/pubsub/nats/mocks.go index 8c356b7b1c..0bc3ada5f0 100644 --- a/router/pkg/pubsub/nats/mocks.go +++ b/router/pkg/pubsub/nats/mocks.go @@ -262,7 +262,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -270,7 +270,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -285,21 +285,21 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context -// - event SubscriptionEventConfiguration +// - event datasource.SubscriptionEventConfiguration // - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 SubscriptionEventConfiguration + var arg1 datasource.SubscriptionEventConfiguration if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) + arg1 = args[1].(datasource.SubscriptionEventConfiguration) } var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { @@ -319,7 +319,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 556e676048..7ede9d6c63 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -13,7 +13,7 @@ import ( // Adapter defines the methods that a Redis adapter should implement type Adapter interface { // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error + Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error // Publish publishes the given event to the specified channel Publish(ctx context.Context, event PublishEventConfiguration) error // Startup initializes the adapter @@ -73,19 +73,23 @@ func (p *ProviderAdapter) Shutdown(ctx context.Context) error { return p.conn.Close() } -func (p *ProviderAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { + subConf, ok := conf.(*SubscriptionEventConfiguration) + if !ok { + return datasource.NewError("invalid event type for Kafka adapter", nil) + } log := p.logger.With( - zap.String("provider_id", event.ProviderID()), + zap.String("provider_id", subConf.ProviderID()), zap.String("method", "subscribe"), - zap.Strings("channels", event.Channels), + zap.Strings("channels", subConf.Channels), ) - sub := p.conn.PSubscribe(ctx, event.Channels...) + sub := p.conn.PSubscribe(ctx, subConf.Channels...) msgChan := sub.Channel() cleanup := func() { - err := sub.PUnsubscribe(ctx, event.Channels...) + err := sub.PUnsubscribe(ctx, subConf.Channels...) if err != nil { - log.Error(fmt.Sprintf("error unsubscribing from redis for topics %v", event.Channels), zap.Error(err)) + log.Error(fmt.Sprintf("error unsubscribing from redis for topics %v", subConf.Channels), zap.Error(err)) } } diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index cb97bafb9a..dae28e93a4 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -7,11 +7,8 @@ import ( "fmt" "io" - "github.com/buger/jsonparser" - "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/httpclient" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) // Event represents an event from Redis @@ -72,61 +69,6 @@ func (s *PublishEventConfiguration) MarshalJSONTemplate() (string, error) { return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s"}`, s.Channel, s.Event.Data, s.ProviderID()), nil } -// SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis -type SubscriptionDataSource struct { - pubSub Adapter -} - -func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { - var subscriptionConfiguration SubscriptionEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return nil - } - return &subscriptionConfiguration -} - -// UniqueRequestID computes a unique ID for the subscription request -func (s *SubscriptionDataSource) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { - val, _, _, err := jsonparser.Get(input, "channels") - if err != nil { - return err - } - - _, err = xxh.Write(val) - if err != nil { - return err - } - - val, _, _, err = jsonparser.Get(input, "providerId") - if err != nil { - return err - } - - _, err = xxh.Write(val) - return err -} - -// Start starts the subscription -func (s *SubscriptionDataSource) Start(ctx *resolve.Context, input []byte, updater datasource.SubscriptionEventUpdater) error { - subConf := s.SubscriptionEventConfiguration(input) - if subConf == nil { - return fmt.Errorf("no subscription configuration found") - } - - conf, ok := subConf.(*SubscriptionEventConfiguration) - if !ok { - return fmt.Errorf("invalid subscription configuration") - } - - return s.pubSub.Subscribe(ctx.Context(), *conf, updater) -} - -// LoadInitialData implements the interface method (not used for this subscription type) -func (s *SubscriptionDataSource) LoadInitialData(ctx context.Context) (initial []byte, err error) { - return nil, nil -} - // PublishDataSource implements resolve.DataSource for Redis publishing type PublishDataSource struct { pubSub Adapter diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index 0db9b5a8f4..bce913e54e 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -5,6 +5,8 @@ import ( "fmt" "slices" + "github.com/buger/jsonparser" + "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) @@ -69,10 +71,28 @@ func (c *EngineDataSourceFactory) ResolveDataSourceInput(eventData []byte) (stri } // ResolveDataSourceSubscription returns the subscription data source -func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.PubSubSubscriptionDataSource, error) { - return &SubscriptionDataSource{ - pubSub: c.RedisAdapter, - }, nil +func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.SubscriptionDataSource, error) { + return datasource.NewPubSubSubscriptionDataSource[*SubscriptionEventConfiguration]( + c.RedisAdapter, + func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + val, _, _, err := jsonparser.Get(input, "channels") + if err != nil { + return err + } + + _, err = xxh.Write(val) + if err != nil { + return err + } + + val, _, _, err = jsonparser.Get(input, "providerId") + if err != nil { + return err + } + + _, err = xxh.Write(val) + return err + }), nil } // ResolveDataSourceSubscriptionInput builds the input for the subscription data source diff --git a/router/pkg/pubsub/redis/engine_datasource_factory_test.go b/router/pkg/pubsub/redis/engine_datasource_factory_test.go index 3d7910cf23..f96691583d 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory_test.go @@ -4,11 +4,15 @@ import ( "bytes" "context" "encoding/json" + "errors" "testing" + "github.com/cespare/xxhash/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router/pkg/pubsub/pubsubtest" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestRedisEngineDataSourceFactory(t *testing.T) { @@ -176,3 +180,57 @@ func TestTransformEventConfig(t *testing.T) { require.Equal(t, []string{"transformed.original.subject1", "transformed.original.subject2"}, cfg.channels) }) } + +func TestRedisEngineDataSourceFactory_UniqueRequestID(t *testing.T) { + tests := []struct { + name string + input string + expectError bool + expectedError error + }{ + { + name: "valid input", + input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, + expectError: false, + }, + { + name: "missing channels", + input: `{"providerId":"test-provider"}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + { + name: "missing providerId", + input: `{"channels":["channel1", "channel2"]}`, + expectError: true, + expectedError: errors.New("Key path not found"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := &EngineDataSourceFactory{ + RedisAdapter: NewMockAdapter(t), + } + source, err := factory.ResolveDataSourceSubscription() + require.NoError(t, err) + ctx := &resolve.Context{} + input := []byte(tt.input) + xxh := xxhash.New() + + err = source.UniqueRequestID(ctx, input, xxh) + + if tt.expectError { + require.Error(t, err) + if tt.expectedError != nil { + // For jsonparser errors, just check if the error message contains the expected text + assert.Contains(t, err.Error(), tt.expectedError.Error()) + } + } else { + require.NoError(t, err) + // Check that the hash has been updated + assert.NotEqual(t, 0, xxh.Sum64()) + } + }) + } +} diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index a343e51503..74b7d564d7 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -7,12 +7,9 @@ import ( "errors" "testing" - "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { @@ -50,124 +47,6 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { } } -func TestSubscriptionSource_UniqueRequestID(t *testing.T) { - tests := []struct { - name string - input string - expectError bool - expectedError error - }{ - { - name: "valid input", - input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, - expectError: false, - }, - { - name: "missing channels", - input: `{"providerId":"test-provider"}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - { - name: "missing providerId", - input: `{"channels":["channel1", "channel2"]}`, - expectError: true, - expectedError: errors.New("Key path not found"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - source := &SubscriptionDataSource{ - pubSub: NewMockAdapter(t), - } - ctx := &resolve.Context{} - input := []byte(tt.input) - xxh := xxhash.New() - - err := source.UniqueRequestID(ctx, input, xxh) - - if tt.expectError { - require.Error(t, err) - if tt.expectedError != nil { - // For jsonparser errors, just check if the error message contains the expected text - assert.Contains(t, err.Error(), tt.expectedError.Error()) - } - } else { - require.NoError(t, err) - // Check that the hash has been updated - assert.NotEqual(t, 0, xxh.Sum64()) - } - }) - } -} - -func TestSubscriptionSource_Start(t *testing.T) { - tests := []struct { - name string - input string - mockSetup func(*MockAdapter, *datasource.MockSubscriptionEventUpdater) - expectError bool - }{ - { - name: "successful subscription", - input: `{"channels":["channel1", "channel2"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Channels: []string{"channel1", "channel2"}, - }, mock.Anything).Return(nil) - }, - expectError: false, - }, - { - name: "adapter returns error", - input: `{"channels":["channel1"], "providerId":"test-provider"}`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) { - m.On("Subscribe", mock.Anything, SubscriptionEventConfiguration{ - Provider: "test-provider", - Channels: []string{"channel1"}, - }, mock.Anything).Return(errors.New("subscription error")) - }, - expectError: true, - }, - { - name: "invalid input json", - input: `{"invalid json":`, - mockSetup: func(m *MockAdapter, updater *datasource.MockSubscriptionEventUpdater) {}, - expectError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockAdapter := NewMockAdapter(t) - updater := datasource.NewMockSubscriptionEventUpdater(t) - tt.mockSetup(mockAdapter, updater) - - source := &SubscriptionDataSource{ - pubSub: mockAdapter, - } - - // Set up go context - goCtx := context.Background() - - // Create a resolve.Context with the standard context - resolveCtx := &resolve.Context{} - resolveCtx = resolveCtx.WithContext(goCtx) - - input := []byte(tt.input) - err := source.Start(resolveCtx, input, updater) - - if tt.expectError { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - func TestRedisPublishDataSource_Load(t *testing.T) { tests := []struct { name string diff --git a/router/pkg/pubsub/redis/mocks.go b/router/pkg/pubsub/redis/mocks.go index 91c6ca9205..6f6938cdd0 100644 --- a/router/pkg/pubsub/redis/mocks.go +++ b/router/pkg/pubsub/redis/mocks.go @@ -198,7 +198,7 @@ func (_c *MockAdapter_Startup_Call) RunAndReturn(run func(ctx context.Context) e } // Subscribe provides a mock function for the type MockAdapter -func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { +func (_mock *MockAdapter) Subscribe(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error { ret := _mock.Called(ctx, event, updater) if len(ret) == 0 { @@ -206,7 +206,7 @@ func (_mock *MockAdapter) Subscribe(ctx context.Context, event SubscriptionEvent } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { + if returnFunc, ok := ret.Get(0).(func(context.Context, datasource.SubscriptionEventConfiguration, datasource.SubscriptionEventUpdater) error); ok { r0 = returnFunc(ctx, event, updater) } else { r0 = ret.Error(0) @@ -221,21 +221,21 @@ type MockAdapter_Subscribe_Call struct { // Subscribe is a helper method to define mock.On call // - ctx context.Context -// - event SubscriptionEventConfiguration +// - event datasource.SubscriptionEventConfiguration // - updater datasource.SubscriptionEventUpdater func (_e *MockAdapter_Expecter) Subscribe(ctx interface{}, event interface{}, updater interface{}) *MockAdapter_Subscribe_Call { return &MockAdapter_Subscribe_Call{Call: _e.mock.On("Subscribe", ctx, event, updater)} } -func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) Run(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater)) *MockAdapter_Subscribe_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { arg0 = args[0].(context.Context) } - var arg1 SubscriptionEventConfiguration + var arg1 datasource.SubscriptionEventConfiguration if args[1] != nil { - arg1 = args[1].(SubscriptionEventConfiguration) + arg1 = args[1].(datasource.SubscriptionEventConfiguration) } var arg2 datasource.SubscriptionEventUpdater if args[2] != nil { @@ -255,7 +255,7 @@ func (_c *MockAdapter_Subscribe_Call) Return(err error) *MockAdapter_Subscribe_C return _c } -func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { +func (_c *MockAdapter_Subscribe_Call) RunAndReturn(run func(ctx context.Context, event datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error) *MockAdapter_Subscribe_Call { _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/redis/provider_builder.go b/router/pkg/pubsub/redis/provider_builder.go index 415963b885..5ea244fa97 100644 --- a/router/pkg/pubsub/redis/provider_builder.go +++ b/router/pkg/pubsub/redis/provider_builder.go @@ -65,7 +65,7 @@ func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventCo }, nil } -// Providers returns the Redis PubSub providers for the given provider IDs +// BuildProvider returns the Redis PubSub providers for the given provider IDs func (b *ProviderBuilder) BuildProvider(provider config.RedisEventSource) (datasource.Provider, error) { adapter := NewProviderAdapter(b.ctx, b.logger, provider.URLs, provider.ClusterEnabled) pubSubProvider := datasource.NewPubSubProvider(provider.ID, providerTypeID, adapter, b.logger) From 85a1bdb0dc9252b08c0bf32639930c31f139e420 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 16:37:15 +0200 Subject: [PATCH 092/173] chore: add a description to PubSubSubscriptionDataSource --- router/pkg/pubsub/datasource/subscription_datasource.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 709d43b2ed..99fecd723d 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -10,6 +10,8 @@ import ( type uniqueRequestIdFn func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error +// PubSubSubscriptionDataSource is a data source for handling subscriptions using a Pub/Sub mechanism. +// It implements the SubscriptionDataSource interface and HookableSubscriptionDataSource type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { pubSub Adapter uniqueRequestID uniqueRequestIdFn @@ -58,6 +60,9 @@ func (s *PubSubSubscriptionDataSource[C]) SetSubscriptionOnStartFns(fns ...Subsc s.subscriptionOnStartFns = append(s.subscriptionOnStartFns, fns...) } +var _ SubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) +var _ resolve.HookableSubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) + func NewPubSubSubscriptionDataSource[C SubscriptionEventConfiguration](pubSub Adapter, uniqueRequestIdFn uniqueRequestIdFn) *PubSubSubscriptionDataSource[C] { return &PubSubSubscriptionDataSource[C]{ pubSub: pubSub, From 27e2b84a623c46d61da431b0691541711dfefafd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 17:45:47 +0200 Subject: [PATCH 093/173] chore: add PubSubSubscriptionDataSource tests --- .../subscription_datasource_test.go | 348 ++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 router/pkg/pubsub/datasource/subscription_datasource_test.go diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go new file mode 100644 index 0000000000..651716178e --- /dev/null +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -0,0 +1,348 @@ +package datasource + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/cespare/xxhash/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +// testSubscriptionEventConfiguration implements SubscriptionEventConfiguration for testing +type testSubscriptionEventConfiguration struct { + Topic string `json:"topic"` + Subject string `json:"subject"` +} + +func (t testSubscriptionEventConfiguration) ProviderID() string { + return "test-provider" +} + +func (t testSubscriptionEventConfiguration) ProviderType() ProviderType { + return ProviderTypeNats +} + +func (t testSubscriptionEventConfiguration) RootFieldName() string { + return "testSubscription" +} + +func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + result := dataSource.SubscriptionEventConfiguration(input) + assert.NotNil(t, result) + + typedResult, ok := result.(testSubscriptionEventConfiguration) + assert.True(t, ok) + assert.Equal(t, "test-topic", typedResult.Topic) + assert.Equal(t, "test-subject", typedResult.Subject) +} + +func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_InvalidJSON(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + invalidInput := []byte(`{"invalid": json}`) + result := dataSource.SubscriptionEventConfiguration(invalidInput) + assert.Nil(t, result) +} + +func TestPubSubSubscriptionDataSource_UniqueRequestID_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + ctx := &resolve.Context{} + input := []byte(`{"test": "data"}`) + xxh := xxhash.New() + + err := dataSource.UniqueRequestID(ctx, input, xxh) + assert.NoError(t, err) +} + +func TestPubSubSubscriptionDataSource_UniqueRequestID_Error(t *testing.T) { + mockAdapter := NewMockProvider(t) + expectedError := errors.New("unique ID generation error") + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return expectedError + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + ctx := &resolve.Context{} + input := []byte(`{"test": "data"}`) + xxh := xxhash.New() + + err := dataSource.UniqueRequestID(ctx, input, xxh) + assert.Error(t, err) + assert.Equal(t, expectedError, err) +} + +func TestPubSubSubscriptionDataSource_Start_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := resolve.NewContext(context.Background()) + mockUpdater := NewMockSubscriptionUpdater(t) + + mockAdapter.On("Subscribe", ctx.Context(), testConfig, mock.AnythingOfType("*datasource.subscriptionEventUpdater")).Return(nil) + + err = dataSource.Start(ctx, input, mockUpdater) + assert.NoError(t, err) + mockAdapter.AssertExpectations(t) +} + +func TestPubSubSubscriptionDataSource_Start_NoConfiguration(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + invalidInput := []byte(`{"invalid": json}`) + ctx := resolve.NewContext(context.Background()) + mockUpdater := NewMockSubscriptionUpdater(t) + + err := dataSource.Start(ctx, invalidInput, mockUpdater) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no subscription configuration found") +} + +func TestPubSubSubscriptionDataSource_Start_SubscribeError(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := resolve.NewContext(context.Background()) + mockUpdater := NewMockSubscriptionUpdater(t) + expectedError := errors.New("subscription error") + + mockAdapter.On("Subscribe", ctx.Context(), testConfig, mock.AnythingOfType("*datasource.subscriptionEventUpdater")).Return(expectedError) + + err = dataSource.Start(ctx, input, mockUpdater) + assert.Error(t, err) + assert.Equal(t, expectedError, err) + mockAdapter.AssertExpectations(t) +} + +func TestPubSubSubscriptionDataSource_SubscriptionOnStart_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := &resolve.Context{} + + close, err := dataSource.SubscriptionOnStart(ctx, input) + assert.NoError(t, err) + assert.False(t, close) +} + +func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + // Add subscription start hooks + hook1Called := false + hook2Called := false + + hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + hook1Called = true + return false, nil + } + + hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + hook2Called = true + return false, nil + } + + dataSource.SetSubscriptionOnStartFns(hook1, hook2) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := &resolve.Context{} + + close, err := dataSource.SubscriptionOnStart(ctx, input) + assert.NoError(t, err) + assert.False(t, close) + assert.True(t, hook1Called) + assert.True(t, hook2Called) +} + +func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsClose(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + // Add hook that returns close=true + hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + return true, nil + } + + dataSource.SetSubscriptionOnStartFns(hook) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := &resolve.Context{} + + close, err := dataSource.SubscriptionOnStart(ctx, input) + assert.NoError(t, err) + assert.True(t, close) +} + +func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + expectedError := errors.New("hook error") + // Add hook that returns an error + hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + return false, expectedError + } + + dataSource.SetSubscriptionOnStartFns(hook) + + testConfig := testSubscriptionEventConfiguration{ + Topic: "test-topic", + Subject: "test-subject", + } + input, err := json.Marshal(testConfig) + assert.NoError(t, err) + + ctx := &resolve.Context{} + + close, err := dataSource.SubscriptionOnStart(ctx, input) + assert.Error(t, err) + assert.Equal(t, expectedError, err) + assert.False(t, close) +} + +func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + // Initially should have no hooks + assert.Len(t, dataSource.subscriptionOnStartFns, 0) + + // Add hooks + hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + return false, nil + } + hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + return false, nil + } + + dataSource.SetSubscriptionOnStartFns(hook1) + assert.Len(t, dataSource.subscriptionOnStartFns, 1) + + dataSource.SetSubscriptionOnStartFns(hook2) + assert.Len(t, dataSource.subscriptionOnStartFns, 2) +} + +func TestNewPubSubSubscriptionDataSource(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + assert.NotNil(t, dataSource) + assert.Equal(t, mockAdapter, dataSource.pubSub) + assert.NotNil(t, dataSource.uniqueRequestID) + assert.Empty(t, dataSource.subscriptionOnStartFns) +} + +func TestPubSubSubscriptionDataSource_InterfaceCompliance(t *testing.T) { + mockAdapter := NewMockProvider(t) + uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { + return nil + } + + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + + // Test that it implements SubscriptionDataSource interface + var _ SubscriptionDataSource = dataSource + + // Test that it implements HookableSubscriptionDataSource interface + var _ resolve.HookableSubscriptionDataSource = dataSource +} From ffbd20934cbb811aedaedab936b5e38c39a4cc67 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 19:39:15 +0200 Subject: [PATCH 094/173] chore: implement PR suggestions --- router/pkg/pubsub/datasource/datasource.go | 2 +- .../datasource/subscription_datasource.go | 22 +++++++++++-------- .../subscription_datasource_test.go | 8 ++++--- router/pkg/pubsub/kafka/engine_datasource.go | 5 ++--- router/pkg/pubsub/nats/engine_datasource.go | 8 +++---- router/pkg/pubsub/pubsub.go | 1 + router/pkg/pubsub/redis/engine_datasource.go | 5 ++--- 7 files changed, 27 insertions(+), 24 deletions(-) diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index b93add32e1..3a3018b745 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -6,7 +6,7 @@ import ( ) type SubscriptionDataSource interface { - SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration + SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 99fecd723d..0ca55fa55e 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -2,7 +2,7 @@ package datasource import ( "encoding/json" - "fmt" + "errors" "github.com/cespare/xxhash/v2" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" @@ -18,13 +18,13 @@ type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { subscriptionOnStartFns []SubscriptionOnStartFn } -func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) SubscriptionEventConfiguration { +func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) { var subscriptionConfiguration C err := json.Unmarshal(input, &subscriptionConfiguration) if err != nil { - return nil + return nil, err } - return subscriptionConfiguration + return subscriptionConfiguration, nil } func (s *PubSubSubscriptionDataSource[C]) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { @@ -32,14 +32,14 @@ func (s *PubSubSubscriptionDataSource[C]) UniqueRequestID(ctx *resolve.Context, } func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error { - subConf := s.SubscriptionEventConfiguration(input) - if subConf == nil { - return fmt.Errorf("no subscription configuration found") + subConf, err := s.SubscriptionEventConfiguration(input) + if err != nil { + return err } conf, ok := subConf.(C) if !ok { - return fmt.Errorf("invalid subscription configuration") + return errors.New("invalid subscription configuration") } return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(updater)) @@ -47,7 +47,11 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { for _, fn := range s.subscriptionOnStartFns { - close, err = fn(ctx, s.SubscriptionEventConfiguration(input)) + conf, errConf := s.SubscriptionEventConfiguration(input) + if errConf != nil { + return true, err + } + close, err = fn(ctx, conf) if err != nil || close { return } diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index 651716178e..bbf0b00608 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -45,7 +45,8 @@ func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_Success(t * input, err := json.Marshal(testConfig) assert.NoError(t, err) - result := dataSource.SubscriptionEventConfiguration(input) + result, err := dataSource.SubscriptionEventConfiguration(input) + assert.NoError(t, err) assert.NotNil(t, result) typedResult, ok := result.(testSubscriptionEventConfiguration) @@ -63,7 +64,8 @@ func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_InvalidJSON dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) invalidInput := []byte(`{"invalid": json}`) - result := dataSource.SubscriptionEventConfiguration(invalidInput) + result, err := dataSource.SubscriptionEventConfiguration(invalidInput) + assert.Error(t, err) assert.Nil(t, result) } @@ -140,7 +142,7 @@ func TestPubSubSubscriptionDataSource_Start_NoConfiguration(t *testing.T) { err := dataSource.Start(ctx, invalidInput, mockUpdater) assert.Error(t, err) - assert.Contains(t, err.Error(), "no subscription configuration found") + assert.Contains(t, err.Error(), "invalid character 'j' looking for beginning of value") } func TestPubSubSubscriptionDataSource_Start_SubscribeError(t *testing.T) { diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index d11b99b5d5..723c0d0bd0 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -87,8 +87,7 @@ type PublishDataSource struct { func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { var publishConfiguration PublishEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) - if err != nil { + if err := json.Unmarshal(input, &publishConfiguration); err != nil { return err } @@ -96,7 +95,7 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B _, err = io.WriteString(out, `{"success": false}`) return err } - _, err = io.WriteString(out, `{"success": true}`) + _, err := io.WriteString(out, `{"success": true}`) return err } diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 5c12d361e3..0fa41e5480 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -83,8 +83,7 @@ type NatsPublishDataSource struct { func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { var publishConfiguration PublishAndRequestEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) - if err != nil { + if err := json.Unmarshal(input, &publishConfiguration); err != nil { return err } @@ -92,7 +91,7 @@ func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *byt _, err = io.WriteString(out, `{"success": false}`) return err } - _, err = io.WriteString(out, `{"success": true}`) + _, err := io.WriteString(out, `{"success": true}`) return err } @@ -106,8 +105,7 @@ type NatsRequestDataSource struct { func (s *NatsRequestDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { var subscriptionConfiguration PublishAndRequestEventConfiguration - err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { + if err := json.Unmarshal(input, &subscriptionConfiguration); err != nil { return err } diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index f8901e3a50..e52e5081bb 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -49,6 +49,7 @@ func (e *ProviderNotDefinedError) Error() string { return fmt.Sprintf("%s provider with ID %s is not defined", e.ProviderTypeID, e.ProviderID) } +// Hooks contains hooks for the pubsub providers and data sources type Hooks struct { SubscriptionOnStart []pubsub_datasource.SubscriptionOnStartFn } diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index dae28e93a4..3a685fe9b0 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -77,8 +77,7 @@ type PublishDataSource struct { // Load processes a request to publish to Redis func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.Buffer) error { var publishConfiguration PublishEventConfiguration - err := json.Unmarshal(input, &publishConfiguration) - if err != nil { + if err := json.Unmarshal(input, &publishConfiguration); err != nil { return err } @@ -86,7 +85,7 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B _, err = io.WriteString(out, `{"success": false}`) return err } - _, err = io.WriteString(out, `{"success": true}`) + _, err := io.WriteString(out, `{"success": true}`) return err } From 37ede740d7986e5ab51e91f9f9de310cd6763c0e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 21:25:28 +0200 Subject: [PATCH 095/173] chore: implement hooks logic in existing subscriptionEventUpdater instead of creating a new one that wraps the first one --- router/core/factoryresolver.go | 2 +- router/pkg/pubsub/datasource/datasource.go | 1 + router/pkg/pubsub/datasource/factory.go | 18 ++--- router/pkg/pubsub/datasource/hooks.go | 20 +++++ router/pkg/pubsub/datasource/planner.go | 3 +- router/pkg/pubsub/datasource/provider.go | 8 -- .../pkg/pubsub/datasource/pubsubprovider.go | 55 +------------- .../datasource/subscription_datasource.go | 7 +- .../datasource/subscription_event_updater.go | 73 ++++++++++++++++--- router/pkg/pubsub/pubsub.go | 13 +--- router/pkg/pubsub/pubsub_test.go | 14 ++-- 11 files changed, 113 insertions(+), 101 deletions(-) create mode 100644 router/pkg/pubsub/datasource/hooks.go diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index 4d93616bf8..eacd8b034f 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -496,7 +496,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod pubSubDS, l.resolver.InstanceData().HostName, l.resolver.InstanceData().ListenAddress, - pubsub.Hooks{ + pubsub_datasource.Hooks{ SubscriptionOnStart: subscriptionOnStartFns, OnStreamEvents: onStreamEventsFns, OnPublishEvents: onPublishEventsFns, diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index 3a3018b745..73b5e9bb71 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -10,6 +10,7 @@ type SubscriptionDataSource interface { Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) + SetOnStreamEventsFns(fns ...OnStreamEventsFn) } // EngineDataSourceFactory is the interface that all pubsub data sources must implement. diff --git a/router/pkg/pubsub/datasource/factory.go b/router/pkg/pubsub/datasource/factory.go index 2431bb21a5..1ad0ac0e23 100644 --- a/router/pkg/pubsub/datasource/factory.go +++ b/router/pkg/pubsub/datasource/factory.go @@ -9,18 +9,18 @@ import ( ) type PlannerConfig[PB ProviderBuilder[P, E], P any, E any] struct { - Providers map[string]Provider - ProviderBuilder PB - Event E - SubscriptionOnStartFns []SubscriptionOnStartFn + Providers map[string]Provider + ProviderBuilder PB + Event E + Hooks Hooks } -func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, providers map[string]Provider, subscriptionOnStartFns []SubscriptionOnStartFn) *PlannerConfig[PB, P, E] { +func NewPlannerConfig[PB ProviderBuilder[P, E], P any, E any](providerBuilder PB, event E, providers map[string]Provider, hooks Hooks) *PlannerConfig[PB, P, E] { return &PlannerConfig[PB, P, E]{ - Providers: providers, - ProviderBuilder: providerBuilder, - Event: event, - SubscriptionOnStartFns: subscriptionOnStartFns, + Providers: providers, + ProviderBuilder: providerBuilder, + Event: event, + Hooks: hooks, } } diff --git a/router/pkg/pubsub/datasource/hooks.go b/router/pkg/pubsub/datasource/hooks.go new file mode 100644 index 0000000000..139328f4d1 --- /dev/null +++ b/router/pkg/pubsub/datasource/hooks.go @@ -0,0 +1,20 @@ +package datasource + +import ( + "context" + + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) + +type OnPublishEventsFn func(ctx context.Context, pubConf PublishEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) + +type OnStreamEventsFn func(ctx context.Context, subConf SubscriptionEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) + +// Hooks contains hooks for the pubsub providers and data sources +type Hooks struct { + SubscriptionOnStart []SubscriptionOnStartFn + OnStreamEvents []OnStreamEventsFn + OnPublishEvents []OnPublishEventsFn +} diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index 2111b294a3..cb572beb80 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -116,7 +116,8 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to get resolve data source subscription: %w", err)) return plan.SubscriptionConfiguration{} } - dataSource.SetSubscriptionOnStartFns(p.config.SubscriptionOnStartFns...) + dataSource.SetSubscriptionOnStartFns(p.config.Hooks.SubscriptionOnStart...) + dataSource.SetOnStreamEventsFns(p.config.Hooks.OnStreamEvents...) input, err := pubSubDataSource.ResolveDataSourceSubscriptionInput() if err != nil { diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 5d44232fe0..039e6655ba 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -2,8 +2,6 @@ package datasource import ( "context" - - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) type ArgumentTemplateCallback func(tpl string) (string, error) @@ -64,12 +62,6 @@ type StreamEvent interface { GetData() []byte } -type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) - -type OnPublishEventsFn func(ctx context.Context, pubConf PublishEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) - -type OnStreamEventsFn func(ctx context.Context, subConf SubscriptionEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) - // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { ProviderID() string diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 4bb8e0accd..94c1bd369e 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -3,7 +3,6 @@ package datasource import ( "context" - "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) @@ -16,51 +15,6 @@ type PubSubProvider struct { Logger *zap.Logger } -type hookedUpdater struct { - ctx context.Context - updater SubscriptionEventUpdater - subscriptionEventConfiguration SubscriptionEventConfiguration - OnStreamEventsFns []OnStreamEventsFn -} - -func (h *hookedUpdater) Update(events []StreamEvent) { - if len(h.OnStreamEventsFns) == 0 { - h.updater.Update(events) - return - } - - processedEvents, err := applyStreamEventHooks(h.ctx, h.subscriptionEventConfiguration, events, h.OnStreamEventsFns) - if err != nil { - // TODO: do something with the error - for now, continue with original events - h.updater.Update(events) - return - } - - h.updater.Update(processedEvents) -} - -func (h *hookedUpdater) Complete() { - h.updater.Complete() -} - -func (h *hookedUpdater) Close(kind resolve.SubscriptionCloseKind) { - h.updater.Close(kind) -} - -// applyStreamEventHooks processes events through a chain of hook functions -// Each hook receives the result from the previous hook, creating a proper middleware pipeline -func applyStreamEventHooks(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent, hooks []OnStreamEventsFn) ([]StreamEvent, error) { - currentEvents := events - for _, hook := range hooks { - var err error - currentEvents, err = hook(ctx, cfg, currentEvents) - if err != nil { - return nil, err - } - } - return currentEvents, nil -} - // applyPublishEventHooks processes events through a chain of hook functions // Each hook receives the result from the previous hook, creating a proper middleware pipeline func applyPublishEventHooks(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent, hooks []OnPublishEventsFn) ([]StreamEvent, error) { @@ -98,14 +52,7 @@ func (p *PubSubProvider) Shutdown(ctx context.Context) error { } func (p *PubSubProvider) Subscribe(ctx context.Context, conf SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { - hookedUpdater := &hookedUpdater{ - ctx: ctx, - updater: updater, - subscriptionEventConfiguration: conf, - OnStreamEventsFns: p.onStreamEventsFns, - } - - return p.Adapter.Subscribe(ctx, conf, hookedUpdater) + return p.Adapter.Subscribe(ctx, conf, updater) } func (p *PubSubProvider) Publish(ctx context.Context, conf PublishEventConfiguration, events []StreamEvent) error { diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 0ca55fa55e..447c7a4cc5 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -16,6 +16,7 @@ type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { pubSub Adapter uniqueRequestID uniqueRequestIdFn subscriptionOnStartFns []SubscriptionOnStartFn + onStreamEventsFns []OnStreamEventsFn } func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) { @@ -42,7 +43,7 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return errors.New("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(updater)) + return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.onStreamEventsFns, updater)) } func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { @@ -64,6 +65,10 @@ func (s *PubSubSubscriptionDataSource[C]) SetSubscriptionOnStartFns(fns ...Subsc s.subscriptionOnStartFns = append(s.subscriptionOnStartFns, fns...) } +func (s *PubSubSubscriptionDataSource[C]) SetOnStreamEventsFns(fns ...OnStreamEventsFn) { + s.onStreamEventsFns = append(s.onStreamEventsFns, fns...) +} + var _ SubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) var _ resolve.HookableSubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index e882a135e7..458d0f733c 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -1,6 +1,10 @@ package datasource -import "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +import ( + "context" + + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) // SubscriptionEventUpdater is a wrapper around the SubscriptionUpdater interface // that provides a way to send the event struct instead of the raw data @@ -9,28 +13,77 @@ type SubscriptionEventUpdater interface { Update(events []StreamEvent) Complete() Close(kind resolve.SubscriptionCloseKind) + SetOnStreamEventsFns([]OnStreamEventsFn) } type subscriptionEventUpdater struct { - eventUpdater resolve.SubscriptionUpdater + eventUpdater resolve.SubscriptionUpdater + ctx context.Context + subscriptionEventConfiguration SubscriptionEventConfiguration + onStreamEventsFns []OnStreamEventsFn } -func (h *subscriptionEventUpdater) Update(events []StreamEvent) { +func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { for _, event := range events { - h.eventUpdater.Update(event.GetData()) + s.eventUpdater.Update(event.GetData()) + } +} + +func (s *subscriptionEventUpdater) Update(events []StreamEvent) { + if len(s.onStreamEventsFns) == 0 { + s.updateEvents(events) + return + } + + processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.onStreamEventsFns) + if err != nil { + // TODO: do something with the error - for now, continue with original events + s.updateEvents(events) + return } + + s.updateEvents(processedEvents) +} + +func (s *subscriptionEventUpdater) Complete() { + s.eventUpdater.Complete() } -func (h *subscriptionEventUpdater) Complete() { - h.eventUpdater.Complete() +func (s *subscriptionEventUpdater) Close(kind resolve.SubscriptionCloseKind) { + s.eventUpdater.Close(kind) } -func (h *subscriptionEventUpdater) Close(kind resolve.SubscriptionCloseKind) { - h.eventUpdater.Close(kind) +func (s *subscriptionEventUpdater) SetOnStreamEventsFns(fns []OnStreamEventsFn) { + s.onStreamEventsFns = fns +} + +// applyStreamEventHooks processes events through a chain of hook functions +// Each hook receives the result from the previous hook, creating a proper middleware pipeline +func applyStreamEventHooks( + ctx context.Context, + cfg SubscriptionEventConfiguration, + events []StreamEvent, + hooks []OnStreamEventsFn) ([]StreamEvent, error) { + currentEvents := events + for _, hook := range hooks { + var err error + currentEvents, err = hook(ctx, cfg, currentEvents) + if err != nil { + return nil, err + } + } + return currentEvents, nil } -func NewSubscriptionEventUpdater(eventUpdater resolve.SubscriptionUpdater) SubscriptionEventUpdater { +func NewSubscriptionEventUpdater( + ctx context.Context, + cfg SubscriptionEventConfiguration, + onStreamEventsFns []OnStreamEventsFn, + eventUpdater resolve.SubscriptionUpdater) SubscriptionEventUpdater { return &subscriptionEventUpdater{ - eventUpdater: eventUpdater, + ctx: ctx, + subscriptionEventConfiguration: cfg, + onStreamEventsFns: onStreamEventsFns, + eventUpdater: eventUpdater, } } diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index 98fcfb3dc4..d1a1d99583 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -49,13 +49,6 @@ func (e *ProviderNotDefinedError) Error() string { return fmt.Sprintf("%s provider with ID %s is not defined", e.ProviderTypeID, e.ProviderID) } -// Hooks contains hooks for the pubsub providers and data sources -type Hooks struct { - SubscriptionOnStart []pubsub_datasource.SubscriptionOnStartFn - OnStreamEvents []pubsub_datasource.OnStreamEventsFn - OnPublishEvents []pubsub_datasource.OnPublishEventsFn -} - // BuildProvidersAndDataSources is a generic function that builds providers and data sources for the given // EventsConfiguration and DataSourceConfigurationWithMetadata func BuildProvidersAndDataSources( @@ -65,7 +58,7 @@ func BuildProvidersAndDataSources( dsConfs []DataSourceConfigurationWithMetadata, hostName string, routerListenAddr string, - hooks Hooks, + hooks pubsub_datasource.Hooks, ) ([]pubsub_datasource.Provider, []plan.DataSource, error) { var pubSubProviders []pubsub_datasource.Provider var outs []plan.DataSource @@ -127,7 +120,7 @@ func BuildProvidersAndDataSources( return pubSubProviders, outs, nil } -func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E], hooks Hooks) (map[string]pubsub_datasource.Provider, []plan.DataSource, error) { +func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder pubsub_datasource.ProviderBuilder[P, E], providersData []P, dsConfs []dsConfAndEvents[E], hooks pubsub_datasource.Hooks) (map[string]pubsub_datasource.Provider, []plan.DataSource, error) { pubSubProviders := make(map[string]pubsub_datasource.Provider) var outs []plan.DataSource @@ -172,7 +165,7 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder builder, event, pubSubProviders, - hooks.SubscriptionOnStart, + hooks, ) out, err := plan.NewDataSourceConfiguration( dsConf.dsConf.Configuration.Id+"-"+builder.TypeID()+"-"+strconv.Itoa(i), diff --git a/router/pkg/pubsub/pubsub_test.go b/router/pkg/pubsub/pubsub_test.go index d35d8bdbef..993a7e3637 100644 --- a/router/pkg/pubsub/pubsub_test.go +++ b/router/pkg/pubsub/pubsub_test.go @@ -67,7 +67,7 @@ func TestBuild_OK(t *testing.T) { // ctx, kafkaBuilder, config.Providers.Kafka, kafkaDsConfsWithEvents // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, datasource.Hooks{}) // Assertions assert.NoError(t, err) @@ -123,7 +123,7 @@ func TestBuild_ProviderError(t *testing.T) { mockBuilder.On("BuildProvider", natsEventSources[0]).Return(nil, errors.New("provider error")) // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, datasource.Hooks{}) // Assertions assert.Error(t, err) @@ -178,7 +178,7 @@ func TestBuild_ShouldGetAnErrorIfProviderIsNotDefined(t *testing.T) { mockBuilder.On("TypeID").Return("nats") // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, datasource.Hooks{}) // Assertions assert.Error(t, err) @@ -243,7 +243,7 @@ func TestBuild_ShouldNotInitializeProviderIfNotUsed(t *testing.T) { mockBuilder.On("BuildProvider", natsEventSources[1]).Return(mockPubSubUsedProvider, nil) // Execute the function - providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, Hooks{}) + providers, dataSources, err := build(ctx, mockBuilder, natsEventSources, dsConfs, datasource.Hooks{}) // Assertions assert.NoError(t, err) @@ -294,7 +294,7 @@ func TestBuildProvidersAndDataSources_Nats_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) + }, zap.NewNop(), dsConfs, "host", "addr", datasource.Hooks{}) // Assertions assert.NoError(t, err) @@ -347,7 +347,7 @@ func TestBuildProvidersAndDataSources_Kafka_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) + }, zap.NewNop(), dsConfs, "host", "addr", datasource.Hooks{}) // Assertions assert.NoError(t, err) @@ -400,7 +400,7 @@ func TestBuildProvidersAndDataSources_Redis_OK(t *testing.T) { {ID: "provider-1"}, }, }, - }, zap.NewNop(), dsConfs, "host", "addr", Hooks{}) + }, zap.NewNop(), dsConfs, "host", "addr", datasource.Hooks{}) // Assertions assert.NoError(t, err) From 202ed52ec94e628fb07524c109d31568f8b833c9 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 30 Jul 2025 21:52:40 +0200 Subject: [PATCH 096/173] chore: revert useless change --- router/pkg/pubsub/datasource/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 039e6655ba..995436a4f2 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -37,7 +37,7 @@ type Provider interface { } // ProviderBuilder is the interface that the provider builder must implement. -type ProviderBuilder[P any, E any] interface { +type ProviderBuilder[P, E any] interface { // TypeID Get the provider type id (e.g. "kafka", "nats") TypeID() string // BuildProvider Build the provider and the adapter From ed22545f6f1d1e2ae019e2373fd50084d0aaf3cc Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 31 Jul 2025 10:02:01 +0200 Subject: [PATCH 097/173] chore: implement suggestion Co-authored-by: Ludwig Bedacht --- router/pkg/pubsub/datasource/subscription_datasource.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 0ca55fa55e..83ce199ff8 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -21,10 +21,7 @@ type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) { var subscriptionConfiguration C err := json.Unmarshal(input, &subscriptionConfiguration) - if err != nil { - return nil, err - } - return subscriptionConfiguration, nil + return subscriptionConfiguration, err } func (s *PubSubSubscriptionDataSource[C]) UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { From d014b3ab1555ee802a159eb67c0d6edc15db0931 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 31 Jul 2025 10:25:20 +0200 Subject: [PATCH 098/173] chore: fix test --- router/pkg/pubsub/datasource/subscription_datasource_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index bbf0b00608..bbeee43037 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -66,7 +66,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_InvalidJSON invalidInput := []byte(`{"invalid": json}`) result, err := dataSource.SubscriptionEventConfiguration(invalidInput) assert.Error(t, err) - assert.Nil(t, result) + assert.Equal(t, testSubscriptionEventConfiguration{}, result) } func TestPubSubSubscriptionDataSource_UniqueRequestID_Success(t *testing.T) { From 8f6adbe3a81f380d4087f2b3c5c506578caec013 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 31 Jul 2025 15:14:25 +0200 Subject: [PATCH 099/173] chore: use hooks instead of specifying single hooks --- router/pkg/pubsub/datasource/datasource.go | 3 +- router/pkg/pubsub/datasource/planner.go | 3 +- .../datasource/subscription_datasource.go | 19 +++++-------- .../subscription_datasource_test.go | 28 +++++++++++++------ .../datasource/subscription_event_updater.go | 16 +++++------ 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index 73b5e9bb71..3cc3732612 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -9,8 +9,7 @@ type SubscriptionDataSource interface { SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) - SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) - SetOnStreamEventsFns(fns ...OnStreamEventsFn) + SetHooks(hoks Hooks) } // EngineDataSourceFactory is the interface that all pubsub data sources must implement. diff --git a/router/pkg/pubsub/datasource/planner.go b/router/pkg/pubsub/datasource/planner.go index cb572beb80..7153590ca3 100644 --- a/router/pkg/pubsub/datasource/planner.go +++ b/router/pkg/pubsub/datasource/planner.go @@ -116,8 +116,7 @@ func (p *Planner[PB, P, E]) ConfigureSubscription() plan.SubscriptionConfigurati p.visitor.Walker.StopWithInternalErr(fmt.Errorf("failed to get resolve data source subscription: %w", err)) return plan.SubscriptionConfiguration{} } - dataSource.SetSubscriptionOnStartFns(p.config.Hooks.SubscriptionOnStart...) - dataSource.SetOnStreamEventsFns(p.config.Hooks.OnStreamEvents...) + dataSource.SetHooks(p.config.Hooks) input, err := pubSubDataSource.ResolveDataSourceSubscriptionInput() if err != nil { diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 447c7a4cc5..dc31095562 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -13,10 +13,9 @@ type uniqueRequestIdFn func(ctx *resolve.Context, input []byte, xxh *xxhash.Dige // PubSubSubscriptionDataSource is a data source for handling subscriptions using a Pub/Sub mechanism. // It implements the SubscriptionDataSource interface and HookableSubscriptionDataSource type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { - pubSub Adapter - uniqueRequestID uniqueRequestIdFn - subscriptionOnStartFns []SubscriptionOnStartFn - onStreamEventsFns []OnStreamEventsFn + pubSub Adapter + uniqueRequestID uniqueRequestIdFn + hooks Hooks } func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) { @@ -43,11 +42,11 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return errors.New("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.onStreamEventsFns, updater)) + return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.hooks, updater)) } func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { - for _, fn := range s.subscriptionOnStartFns { + for _, fn := range s.hooks.SubscriptionOnStart { conf, errConf := s.SubscriptionEventConfiguration(input) if errConf != nil { return true, err @@ -61,12 +60,8 @@ func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Conte return } -func (s *PubSubSubscriptionDataSource[C]) SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) { - s.subscriptionOnStartFns = append(s.subscriptionOnStartFns, fns...) -} - -func (s *PubSubSubscriptionDataSource[C]) SetOnStreamEventsFns(fns ...OnStreamEventsFn) { - s.onStreamEventsFns = append(s.onStreamEventsFns, fns...) +func (s *PubSubSubscriptionDataSource[C]) SetHooks(hooks Hooks) { + s.hooks = hooks } var _ SubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index bbf0b00608..8b2e75eb8e 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -216,7 +216,9 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T return false, nil } - dataSource.SetSubscriptionOnStartFns(hook1, hook2) + dataSource.SetHooks(Hooks{ + SubscriptionOnStart: []SubscriptionOnStartFn{hook1, hook2}, + }) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -247,7 +249,9 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsClose(t *te return true, nil } - dataSource.SetSubscriptionOnStartFns(hook) + dataSource.SetHooks(Hooks{ + SubscriptionOnStart: []SubscriptionOnStartFn{hook}, + }) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -277,7 +281,9 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te return false, expectedError } - dataSource.SetSubscriptionOnStartFns(hook) + dataSource.SetHooks(Hooks{ + SubscriptionOnStart: []SubscriptionOnStartFn{hook}, + }) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -303,7 +309,7 @@ func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) // Initially should have no hooks - assert.Len(t, dataSource.subscriptionOnStartFns, 0) + assert.Len(t, dataSource.hooks.SubscriptionOnStart, 0) // Add hooks hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { @@ -313,11 +319,15 @@ func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { return false, nil } - dataSource.SetSubscriptionOnStartFns(hook1) - assert.Len(t, dataSource.subscriptionOnStartFns, 1) + dataSource.SetHooks(Hooks{ + SubscriptionOnStart: []SubscriptionOnStartFn{hook1}, + }) + assert.Len(t, dataSource.hooks.SubscriptionOnStart, 1) - dataSource.SetSubscriptionOnStartFns(hook2) - assert.Len(t, dataSource.subscriptionOnStartFns, 2) + dataSource.SetHooks(Hooks{ + SubscriptionOnStart: []SubscriptionOnStartFn{hook2}, + }) + assert.Len(t, dataSource.hooks.SubscriptionOnStart, 1) } func TestNewPubSubSubscriptionDataSource(t *testing.T) { @@ -331,7 +341,7 @@ func TestNewPubSubSubscriptionDataSource(t *testing.T) { assert.NotNil(t, dataSource) assert.Equal(t, mockAdapter, dataSource.pubSub) assert.NotNil(t, dataSource.uniqueRequestID) - assert.Empty(t, dataSource.subscriptionOnStartFns) + assert.Empty(t, dataSource.hooks.SubscriptionOnStart) } func TestPubSubSubscriptionDataSource_InterfaceCompliance(t *testing.T) { diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 458d0f733c..b18968840e 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -13,14 +13,14 @@ type SubscriptionEventUpdater interface { Update(events []StreamEvent) Complete() Close(kind resolve.SubscriptionCloseKind) - SetOnStreamEventsFns([]OnStreamEventsFn) + SetHooks(hooks Hooks) } type subscriptionEventUpdater struct { eventUpdater resolve.SubscriptionUpdater ctx context.Context subscriptionEventConfiguration SubscriptionEventConfiguration - onStreamEventsFns []OnStreamEventsFn + hooks Hooks } func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { @@ -30,12 +30,12 @@ func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { } func (s *subscriptionEventUpdater) Update(events []StreamEvent) { - if len(s.onStreamEventsFns) == 0 { + if len(s.hooks.OnStreamEvents) == 0 { s.updateEvents(events) return } - processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.onStreamEventsFns) + processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnStreamEvents) if err != nil { // TODO: do something with the error - for now, continue with original events s.updateEvents(events) @@ -53,8 +53,8 @@ func (s *subscriptionEventUpdater) Close(kind resolve.SubscriptionCloseKind) { s.eventUpdater.Close(kind) } -func (s *subscriptionEventUpdater) SetOnStreamEventsFns(fns []OnStreamEventsFn) { - s.onStreamEventsFns = fns +func (s *subscriptionEventUpdater) SetHooks(hooks Hooks) { + s.hooks = hooks } // applyStreamEventHooks processes events through a chain of hook functions @@ -78,12 +78,12 @@ func applyStreamEventHooks( func NewSubscriptionEventUpdater( ctx context.Context, cfg SubscriptionEventConfiguration, - onStreamEventsFns []OnStreamEventsFn, + hooks Hooks, eventUpdater resolve.SubscriptionUpdater) SubscriptionEventUpdater { return &subscriptionEventUpdater{ ctx: ctx, subscriptionEventConfiguration: cfg, - onStreamEventsFns: onStreamEventsFns, + hooks: hooks, eventUpdater: eventUpdater, } } From 98c5bd0932dbae109b2962e2619338ead0a30b02 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 31 Jul 2025 17:06:45 +0200 Subject: [PATCH 100/173] chore: better names --- router/pkg/pubsub/datasource/pubsubprovider.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 94c1bd369e..7d2f5b8b63 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -7,12 +7,12 @@ import ( ) type PubSubProvider struct { - onPublishEventsFns []OnPublishEventsFn - onStreamEventsFns []OnStreamEventsFn id string typeID string Adapter Adapter Logger *zap.Logger + onPublishEventsFns []OnPublishEventsFn + onStreamEventsFns []OnStreamEventsFn } // applyPublishEventHooks processes events through a chain of hook functions @@ -51,21 +51,21 @@ func (p *PubSubProvider) Shutdown(ctx context.Context) error { return nil } -func (p *PubSubProvider) Subscribe(ctx context.Context, conf SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { - return p.Adapter.Subscribe(ctx, conf, updater) +func (p *PubSubProvider) Subscribe(ctx context.Context, cfg SubscriptionEventConfiguration, updater SubscriptionEventUpdater) error { + return p.Adapter.Subscribe(ctx, cfg, updater) } -func (p *PubSubProvider) Publish(ctx context.Context, conf PublishEventConfiguration, events []StreamEvent) error { +func (p *PubSubProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { if len(p.onPublishEventsFns) == 0 { - return p.Adapter.Publish(ctx, conf, events) + return p.Adapter.Publish(ctx, cfg, events) } - processedEvents, err := applyPublishEventHooks(ctx, conf, events, p.onPublishEventsFns) + processedEvents, err := applyPublishEventHooks(ctx, cfg, events, p.onPublishEventsFns) if err != nil { return err } - return p.Adapter.Publish(ctx, conf, processedEvents) + return p.Adapter.Publish(ctx, cfg, processedEvents) } func (p *PubSubProvider) SetOnPublishEventsFns(fns []OnPublishEventsFn) { From 7e36d3e7aebc293a164f5ba50a8f2b9aa57b753e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 31 Jul 2025 18:16:46 +0200 Subject: [PATCH 101/173] feat: manage error on update --- router-tests/events/utils.go | 3 ++- .../datasource/subscription_event_updater.go | 12 +++++------ router/pkg/pubsub/kafka/adapter.go | 6 +++++- router/pkg/pubsub/nats/adapter.go | 21 +++++++++++++++++-- router/pkg/pubsub/redis/adapter.go | 8 ++++++- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/router-tests/events/utils.go b/router-tests/events/utils.go index 071bc92128..406df3e560 100644 --- a/router-tests/events/utils.go +++ b/router-tests/events/utils.go @@ -14,7 +14,7 @@ func EnsureTopicExists(t *testing.T, xEnv *testenv.Environment, timeout time.Dur // Delete topic for idempotency deleteCtx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - prefixedTopics := make([]string, len(topics)) + prefixedTopics := make([]string, 0, len(topics)) for _, topic := range topics { prefixedTopics = append(prefixedTopics, xEnv.GetPubSubName(topic)) } @@ -61,6 +61,7 @@ func ReadKafkaMessages(xEnv *testenv.Environment, timeout time.Duration, topicNa if err != nil { return nil, err } + defer client.Close() fetchs := client.PollRecords(ctx, msgs) diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index b18968840e..1e107d2357 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -10,7 +10,7 @@ import ( // that provides a way to send the event struct instead of the raw data // It is used to give access to the event additional fields to the hooks. type SubscriptionEventUpdater interface { - Update(events []StreamEvent) + Update(events []StreamEvent) error Complete() Close(kind resolve.SubscriptionCloseKind) SetHooks(hooks Hooks) @@ -29,20 +29,20 @@ func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { } } -func (s *subscriptionEventUpdater) Update(events []StreamEvent) { +func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { if len(s.hooks.OnStreamEvents) == 0 { s.updateEvents(events) - return + return nil } processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnStreamEvents) if err != nil { - // TODO: do something with the error - for now, continue with original events - s.updateEvents(events) - return + return err } s.updateEvents(processedEvents) + + return nil } func (s *subscriptionEventUpdater) Complete() { diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index 4d62fa8371..b23b3d65f3 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -87,11 +87,15 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u headers[header.Key] = header.Value } - updater.Update([]datasource.StreamEvent{&Event{ + err := updater.Update([]datasource.StreamEvent{&Event{ Data: r.Value, Headers: headers, Key: r.Key, }}) + // if an error occurred while updating the subscription, should exit the poller + if err != nil { + return err + } } } } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index a4f69d4039..75e4133c18 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -132,10 +132,15 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) - updater.Update([]datasource.StreamEvent{&Event{ + updateErr := updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data(), Headers: msg.Headers(), }}) + if updateErr != nil { + // If the update fails, we do not acknowledge the message + log.Error("error updating subscription", zap.Error(updateErr), zap.String("message_subject", msg.Subject())) + return + } // Acknowledge the message after it has been processed ackErr := msg.Ack() @@ -172,10 +177,22 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip select { case msg := <-msgChan: log.Debug("subscription update", zap.String("message_subject", msg.Subject), zap.ByteString("data", msg.Data)) - updater.Update([]datasource.StreamEvent{&Event{ + updateErr := updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data, Headers: msg.Header, }}) + if updateErr != nil { + // If the update fails, we log the error and unsubscribe from all subscriptions + log.Error("error updating subscription", zap.Error(updateErr), zap.String("message_subject", msg.Subject)) + for _, subscription := range subscriptions { + if err := subscription.Unsubscribe(); err != nil { + log.Error("unsubscribing from NATS subject after an error on updating subscription", + zap.Error(err), zap.String("subject", subscription.Subject), + ) + } + } + return + } case <-p.ctx.Done(): // When the application context is done, we stop the subscriptions for _, subscription := range subscriptions { diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 0c844b4d93..67a3f1542b 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -114,9 +114,15 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri return } log.Debug("subscription update", zap.String("message_channel", msg.Channel), zap.String("data", msg.Payload)) - updater.Update([]datasource.StreamEvent{&Event{ + updateErr := updater.Update([]datasource.StreamEvent{&Event{ Data: []byte(msg.Payload), }}) + if updateErr != nil { + log.Error("error updating subscription, stopping subscription", zap.Error(updateErr), zap.String("message_channel", msg.Channel)) + // If the error is not recoverable, we stop the subscription + cleanup() + return + } case <-p.ctx.Done(): // When the application context is done, we stop the subscription if it is not already done log.Debug("application context done, stopping subscription") From 75932b937eddcdebedeb79fbdd232fa5c75833bd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 1 Aug 2025 12:20:46 +0200 Subject: [PATCH 102/173] chore: use hooks on providers --- router/pkg/pubsub/datasource/provider.go | 6 ++--- .../pkg/pubsub/datasource/pubsubprovider.go | 23 ++++++++----------- router/pkg/pubsub/pubsub.go | 3 +-- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 995436a4f2..b9f5903485 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -30,10 +30,8 @@ type Provider interface { ID() string // TypeID Get the provider type id (e.g. "kafka", "nats") TypeID() string - // SetOnPublishEventsFns Set the functions that will be called before publishing events - SetOnPublishEventsFns([]OnPublishEventsFn) - // SetOnStreamEventsFns Set the functions that will be called when receiving events - SetOnStreamEventsFns([]OnStreamEventsFn) + // SetHooks Set the hooks + SetHooks(Hooks) } // ProviderBuilder is the interface that the provider builder must implement. diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 7d2f5b8b63..2d9514d5e8 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -7,12 +7,11 @@ import ( ) type PubSubProvider struct { - id string - typeID string - Adapter Adapter - Logger *zap.Logger - onPublishEventsFns []OnPublishEventsFn - onStreamEventsFns []OnStreamEventsFn + id string + typeID string + Adapter Adapter + Logger *zap.Logger + hooks Hooks } // applyPublishEventHooks processes events through a chain of hook functions @@ -56,11 +55,11 @@ func (p *PubSubProvider) Subscribe(ctx context.Context, cfg SubscriptionEventCon } func (p *PubSubProvider) Publish(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) error { - if len(p.onPublishEventsFns) == 0 { + if len(p.hooks.OnPublishEvents) == 0 { return p.Adapter.Publish(ctx, cfg, events) } - processedEvents, err := applyPublishEventHooks(ctx, cfg, events, p.onPublishEventsFns) + processedEvents, err := applyPublishEventHooks(ctx, cfg, events, p.hooks.OnPublishEvents) if err != nil { return err } @@ -68,12 +67,8 @@ func (p *PubSubProvider) Publish(ctx context.Context, cfg PublishEventConfigurat return p.Adapter.Publish(ctx, cfg, processedEvents) } -func (p *PubSubProvider) SetOnPublishEventsFns(fns []OnPublishEventsFn) { - p.onPublishEventsFns = fns -} - -func (p *PubSubProvider) SetOnStreamEventsFns(fns []OnStreamEventsFn) { - p.onStreamEventsFns = fns +func (p *PubSubProvider) SetHooks(hooks Hooks) { + p.hooks = hooks } func NewPubSubProvider(id string, typeID string, adapter Adapter, logger *zap.Logger) *PubSubProvider { diff --git a/router/pkg/pubsub/pubsub.go b/router/pkg/pubsub/pubsub.go index d1a1d99583..fe86f65059 100644 --- a/router/pkg/pubsub/pubsub.go +++ b/router/pkg/pubsub/pubsub.go @@ -143,8 +143,7 @@ func build[P GetID, E GetEngineEventConfiguration](ctx context.Context, builder if err != nil { return nil, nil, err } - provider.SetOnStreamEventsFns(hooks.OnStreamEvents) - provider.SetOnPublishEventsFns(hooks.OnPublishEvents) + provider.SetHooks(hooks) pubSubProviders[provider.ID()] = provider } From 7bbec8fbc7c00361ecf4ec720a54fb5d86cbc1ca Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 1 Aug 2025 17:42:54 +0200 Subject: [PATCH 103/173] chore: add tests --- router/pkg/pubsub/datasource/mocks.go | 133 +++--- .../pubsub/datasource/pubsubprovider_test.go | 418 +++++++++++++++++- 2 files changed, 482 insertions(+), 69 deletions(-) diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index 332d8ddac6..764c28404e 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -619,28 +619,28 @@ func (_c *MockProvider_Publish_Call) RunAndReturn(run func(ctx context.Context, return _c } -// SetOnPublishEventsFns provides a mock function for the type MockProvider -func (_mock *MockProvider) SetOnPublishEventsFns(onPublishEventsFns []OnPublishEventsFn) { - _mock.Called(onPublishEventsFns) +// SetHooks provides a mock function for the type MockProvider +func (_mock *MockProvider) SetHooks(hooks Hooks) { + _mock.Called(hooks) return } -// MockProvider_SetOnPublishEventsFns_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetOnPublishEventsFns' -type MockProvider_SetOnPublishEventsFns_Call struct { +// MockProvider_SetHooks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetHooks' +type MockProvider_SetHooks_Call struct { *mock.Call } -// SetOnPublishEventsFns is a helper method to define mock.On call -// - onPublishEventsFns []OnPublishEventsFn -func (_e *MockProvider_Expecter) SetOnPublishEventsFns(onPublishEventsFns interface{}) *MockProvider_SetOnPublishEventsFns_Call { - return &MockProvider_SetOnPublishEventsFns_Call{Call: _e.mock.On("SetOnPublishEventsFns", onPublishEventsFns)} +// SetHooks is a helper method to define mock.On call +// - hooks Hooks +func (_e *MockProvider_Expecter) SetHooks(hooks interface{}) *MockProvider_SetHooks_Call { + return &MockProvider_SetHooks_Call{Call: _e.mock.On("SetHooks", hooks)} } -func (_c *MockProvider_SetOnPublishEventsFns_Call) Run(run func(onPublishEventsFns []OnPublishEventsFn)) *MockProvider_SetOnPublishEventsFns_Call { +func (_c *MockProvider_SetHooks_Call) Run(run func(hooks Hooks)) *MockProvider_SetHooks_Call { _c.Call.Run(func(args mock.Arguments) { - var arg0 []OnPublishEventsFn + var arg0 Hooks if args[0] != nil { - arg0 = args[0].([]OnPublishEventsFn) + arg0 = args[0].(Hooks) } run( arg0, @@ -649,52 +649,12 @@ func (_c *MockProvider_SetOnPublishEventsFns_Call) Run(run func(onPublishEventsF return _c } -func (_c *MockProvider_SetOnPublishEventsFns_Call) Return() *MockProvider_SetOnPublishEventsFns_Call { +func (_c *MockProvider_SetHooks_Call) Return() *MockProvider_SetHooks_Call { _c.Call.Return() return _c } -func (_c *MockProvider_SetOnPublishEventsFns_Call) RunAndReturn(run func(onPublishEventsFns []OnPublishEventsFn)) *MockProvider_SetOnPublishEventsFns_Call { - _c.Run(run) - return _c -} - -// SetOnStreamEventsFns provides a mock function for the type MockProvider -func (_mock *MockProvider) SetOnStreamEventsFns(onStreamEventsFns []OnStreamEventsFn) { - _mock.Called(onStreamEventsFns) - return -} - -// MockProvider_SetOnStreamEventsFns_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetOnStreamEventsFns' -type MockProvider_SetOnStreamEventsFns_Call struct { - *mock.Call -} - -// SetOnStreamEventsFns is a helper method to define mock.On call -// - onStreamEventsFns []OnStreamEventsFn -func (_e *MockProvider_Expecter) SetOnStreamEventsFns(onStreamEventsFns interface{}) *MockProvider_SetOnStreamEventsFns_Call { - return &MockProvider_SetOnStreamEventsFns_Call{Call: _e.mock.On("SetOnStreamEventsFns", onStreamEventsFns)} -} - -func (_c *MockProvider_SetOnStreamEventsFns_Call) Run(run func(onStreamEventsFns []OnStreamEventsFn)) *MockProvider_SetOnStreamEventsFns_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 []OnStreamEventsFn - if args[0] != nil { - arg0 = args[0].([]OnStreamEventsFn) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockProvider_SetOnStreamEventsFns_Call) Return() *MockProvider_SetOnStreamEventsFns_Call { - _c.Call.Return() - return _c -} - -func (_c *MockProvider_SetOnStreamEventsFns_Call) RunAndReturn(run func(onStreamEventsFns []OnStreamEventsFn)) *MockProvider_SetOnStreamEventsFns_Call { +func (_c *MockProvider_SetHooks_Call) RunAndReturn(run func(hooks Hooks)) *MockProvider_SetHooks_Call { _c.Run(run) return _c } @@ -1209,12 +1169,63 @@ func (_c *MockSubscriptionEventUpdater_Complete_Call) RunAndReturn(run func()) * return _c } -// Update provides a mock function for the type MockSubscriptionEventUpdater -func (_mock *MockSubscriptionEventUpdater) Update(events []StreamEvent) { - _mock.Called(events) +// SetHooks provides a mock function for the type MockSubscriptionEventUpdater +func (_mock *MockSubscriptionEventUpdater) SetHooks(hooks Hooks) { + _mock.Called(hooks) return } +// MockSubscriptionEventUpdater_SetHooks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetHooks' +type MockSubscriptionEventUpdater_SetHooks_Call struct { + *mock.Call +} + +// SetHooks is a helper method to define mock.On call +// - hooks Hooks +func (_e *MockSubscriptionEventUpdater_Expecter) SetHooks(hooks interface{}) *MockSubscriptionEventUpdater_SetHooks_Call { + return &MockSubscriptionEventUpdater_SetHooks_Call{Call: _e.mock.On("SetHooks", hooks)} +} + +func (_c *MockSubscriptionEventUpdater_SetHooks_Call) Run(run func(hooks Hooks)) *MockSubscriptionEventUpdater_SetHooks_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 Hooks + if args[0] != nil { + arg0 = args[0].(Hooks) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockSubscriptionEventUpdater_SetHooks_Call) Return() *MockSubscriptionEventUpdater_SetHooks_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionEventUpdater_SetHooks_Call) RunAndReturn(run func(hooks Hooks)) *MockSubscriptionEventUpdater_SetHooks_Call { + _c.Run(run) + return _c +} + +// Update provides a mock function for the type MockSubscriptionEventUpdater +func (_mock *MockSubscriptionEventUpdater) Update(events []StreamEvent) error { + ret := _mock.Called(events) + + if len(ret) == 0 { + panic("no return value specified for Update") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func([]StreamEvent) error); ok { + r0 = returnFunc(events) + } else { + r0 = ret.Error(0) + } + return r0 +} + // MockSubscriptionEventUpdater_Update_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Update' type MockSubscriptionEventUpdater_Update_Call struct { *mock.Call @@ -1239,12 +1250,12 @@ func (_c *MockSubscriptionEventUpdater_Update_Call) Run(run func(events []Stream return _c } -func (_c *MockSubscriptionEventUpdater_Update_Call) Return() *MockSubscriptionEventUpdater_Update_Call { - _c.Call.Return() +func (_c *MockSubscriptionEventUpdater_Update_Call) Return(err error) *MockSubscriptionEventUpdater_Update_Call { + _c.Call.Return(err) return _c } -func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(events []StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { - _c.Run(run) +func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(events []StreamEvent) error) *MockSubscriptionEventUpdater_Update_Call { + _c.Call.Return(run) return _c } diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index 134bfbd6bb..f2618b0d06 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -7,8 +7,54 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.uber.org/zap" ) +// Test helper types +type testEvent struct { + data []byte +} + +func (e *testEvent) GetData() []byte { + return e.data +} + +type testSubscriptionConfig struct { + providerID string + providerType ProviderType + fieldName string +} + +func (c *testSubscriptionConfig) ProviderID() string { + return c.providerID +} + +func (c *testSubscriptionConfig) ProviderType() ProviderType { + return c.providerType +} + +func (c *testSubscriptionConfig) RootFieldName() string { + return c.fieldName +} + +type testPublishConfig struct { + providerID string + providerType ProviderType + fieldName string +} + +func (c *testPublishConfig) ProviderID() string { + return c.providerID +} + +func (c *testPublishConfig) ProviderType() ProviderType { + return c.providerType +} + +func (c *testPublishConfig) RootFieldName() string { + return c.fieldName +} + func TestProvider_Startup_Success(t *testing.T) { mockAdapter := NewMockProvider(t) mockAdapter.On("Startup", mock.Anything).Return(nil) @@ -57,18 +103,374 @@ func TestProvider_Shutdown_Error(t *testing.T) { assert.Error(t, err) } -func TestProvider_ID(t *testing.T) { - const testID = "test-id" +func TestProvider_Subscribe_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + mockUpdater := NewMockSubscriptionEventUpdater(t) + config := &testSubscriptionConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + + mockAdapter.On("Subscribe", mock.Anything, config, mockUpdater).Return(nil) + + provider := PubSubProvider{ + Adapter: mockAdapter, + } + err := provider.Subscribe(context.Background(), config, mockUpdater) + + assert.NoError(t, err) +} + +func TestProvider_Subscribe_Error(t *testing.T) { + mockAdapter := NewMockProvider(t) + mockUpdater := NewMockSubscriptionEventUpdater(t) + config := &testSubscriptionConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + expectedError := errors.New("subscription error") + + mockAdapter.On("Subscribe", mock.Anything, config, mockUpdater).Return(expectedError) + provider := PubSubProvider{ - id: testID, + Adapter: mockAdapter, } - assert.Equal(t, testID, provider.ID()) + err := provider.Subscribe(context.Background(), config, mockUpdater) + + assert.Error(t, err) + assert.Equal(t, expectedError, err) } -func TestProvider_TypeID(t *testing.T) { - const providerTypeID = "test-type-id" +func TestProvider_Publish_NoHooks_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data 1")}, + &testEvent{data: []byte("test data 2")}, + } + + mockAdapter.On("Publish", mock.Anything, config, events).Return(nil) + provider := PubSubProvider{ - typeID: providerTypeID, + Adapter: mockAdapter, + hooks: Hooks{}, // No hooks } - assert.Equal(t, providerTypeID, provider.TypeID()) + err := provider.Publish(context.Background(), config, events) + + assert.NoError(t, err) +} + +func TestProvider_Publish_NoHooks_Error(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + expectedError := errors.New("publish error") + + mockAdapter.On("Publish", mock.Anything, config, events).Return(expectedError) + + provider := PubSubProvider{ + Adapter: mockAdapter, + hooks: Hooks{}, // No hooks + } + err := provider.Publish(context.Background(), config, events) + + assert.Error(t, err) + assert.Equal(t, expectedError, err) +} + +func TestProvider_Publish_WithHooks_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original data")}, + } + modifiedEvents := []StreamEvent{ + &testEvent{data: []byte("modified data")}, + } + + // Define hook that modifies events + testHook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return modifiedEvents, nil + } + + mockAdapter.On("Publish", mock.Anything, config, modifiedEvents).Return(nil) + + provider := PubSubProvider{ + Adapter: mockAdapter, + hooks: Hooks{ + OnPublishEvents: []OnPublishEventsFn{testHook}, + }, + } + err := provider.Publish(context.Background(), config, originalEvents) + + assert.NoError(t, err) +} + +func TestProvider_Publish_WithHooks_HookError(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + hookError := errors.New("hook processing error") + + // Define hook that returns an error + testHook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, hookError + } + + // Should not call Publish on adapter since hook fails + provider := PubSubProvider{ + Adapter: mockAdapter, + hooks: Hooks{ + OnPublishEvents: []OnPublishEventsFn{testHook}, + }, + } + err := provider.Publish(context.Background(), config, events) + + assert.Error(t, err) + assert.Equal(t, hookError, err) + // Assert that Publish was not called on the adapter + mockAdapter.AssertNotCalled(t, "Publish") +} + +func TestProvider_Publish_WithHooks_AdapterError(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original data")}, + } + processedEvents := []StreamEvent{ + &testEvent{data: []byte("processed data")}, + } + adapterError := errors.New("adapter publish error") + + // Define hook that processes events successfully + testHook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return processedEvents, nil + } + + mockAdapter.On("Publish", mock.Anything, config, processedEvents).Return(adapterError) + + provider := PubSubProvider{ + Adapter: mockAdapter, + hooks: Hooks{ + OnPublishEvents: []OnPublishEventsFn{testHook}, + }, + } + err := provider.Publish(context.Background(), config, originalEvents) + + assert.Error(t, err) + assert.Equal(t, adapterError, err) +} + +func TestProvider_Publish_WithMultipleHooks_Success(t *testing.T) { + mockAdapter := NewMockProvider(t) + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + + // Chain of hooks that modify the data + hook1 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("modified by hook1")}}, nil + } + hook2 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("modified by hook2")}}, nil + } + + mockAdapter.On("Publish", mock.Anything, config, mock.MatchedBy(func(events []StreamEvent) bool { + return len(events) == 1 && string(events[0].GetData()) == "modified by hook2" + })).Return(nil) + + provider := PubSubProvider{ + Adapter: mockAdapter, + hooks: Hooks{ + OnPublishEvents: []OnPublishEventsFn{hook1, hook2}, + }, + } + err := provider.Publish(context.Background(), config, originalEvents) + + assert.NoError(t, err) +} + +func TestProvider_SetHooks(t *testing.T) { + provider := &PubSubProvider{} + + testHook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return events, nil + } + + hooks := Hooks{ + OnPublishEvents: []OnPublishEventsFn{testHook}, + } + + provider.SetHooks(hooks) + + assert.Equal(t, hooks, provider.hooks) +} + +func TestNewPubSubProvider(t *testing.T) { + mockAdapter := NewMockProvider(t) + logger := zap.NewNop() + id := "test-provider-id" + typeID := "test-type-id" + + provider := NewPubSubProvider(id, typeID, mockAdapter, logger) + + assert.NotNil(t, provider) + assert.Equal(t, id, provider.ID()) + assert.Equal(t, typeID, provider.TypeID()) + assert.Equal(t, mockAdapter, provider.Adapter) + assert.Equal(t, logger, provider.Logger) + assert.Empty(t, provider.hooks.OnPublishEvents) +} + +func TestApplyPublishEventHooks_NoHooks(t *testing.T) { + ctx := context.Background() + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + + result, err := applyPublishEventHooks(ctx, config, originalEvents, []OnPublishEventsFn{}) + + assert.NoError(t, err) + assert.Equal(t, originalEvents, result) +} + +func TestApplyPublishEventHooks_SingleHook_Success(t *testing.T) { + ctx := context.Background() + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + modifiedEvents := []StreamEvent{ + &testEvent{data: []byte("modified")}, + } + + hook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return modifiedEvents, nil + } + + result, err := applyPublishEventHooks(ctx, config, originalEvents, []OnPublishEventsFn{hook}) + + assert.NoError(t, err) + assert.Equal(t, modifiedEvents, result) +} + +func TestApplyPublishEventHooks_SingleHook_Error(t *testing.T) { + ctx := context.Background() + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + hookError := errors.New("hook processing failed") + + hook := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, hookError + } + + result, err := applyPublishEventHooks(ctx, config, originalEvents, []OnPublishEventsFn{hook}) + + assert.Error(t, err) + assert.Equal(t, hookError, err) + assert.Nil(t, result) +} + +func TestApplyPublishEventHooks_MultipleHooks_Success(t *testing.T) { + ctx := context.Background() + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + + hook1 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("step1")}}, nil + } + hook2 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("step2")}}, nil + } + hook3 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("final")}}, nil + } + + result, err := applyPublishEventHooks(ctx, config, originalEvents, []OnPublishEventsFn{hook1, hook2, hook3}) + + assert.NoError(t, err) + assert.Len(t, result, 1) + assert.Equal(t, "final", string(result[0].GetData())) +} + +func TestApplyPublishEventHooks_MultipleHooks_MiddleHookError(t *testing.T) { + ctx := context.Background() + config := &testPublishConfig{ + providerID: "test-provider", + providerType: ProviderTypeKafka, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + middleHookError := errors.New("middle hook failed") + + hook1 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("step1")}}, nil + } + hook2 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, middleHookError + } + hook3 := func(ctx context.Context, cfg PublishEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return []StreamEvent{&testEvent{data: []byte("final")}}, nil + } + + result, err := applyPublishEventHooks(ctx, config, originalEvents, []OnPublishEventsFn{hook1, hook2, hook3}) + + assert.Error(t, err) + assert.Equal(t, middleHookError, err) + assert.Nil(t, result) } From 82b552171d450b0f05684056642b6a6dfbd33496 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 4 Aug 2025 10:06:38 +0200 Subject: [PATCH 104/173] chore: update go.mod --- demo/go.mod | 4 ++-- demo/go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index d1c2b5907c..904b5bd2df 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f + github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index abd0ad07d9..06e1c65526 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -282,8 +282,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= -github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E= +github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f h1:ZxrYipVu20RKNWAMduseyExlZKl7uOXoNsIFtxQrcwo= -github.com/wundergraph/cosmo/router v0.0.0-20250718080012-21acbd83c77f/go.mod h1:2ORZCPYHJS5IqKnJztnPkMzyPf2U+NV6gi4JAYOOG80= +github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 h1:+n2t7+lXsqR85wa7IANB6y8JdSoUOrzwSxFhJMrgVcQ= +github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555/go.mod h1:QapzpsmO9xKjIft1R3+tVl8dAUu0ZvDyOZnfwRmqRZc= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207 h1:g2MpMjU/Jk30oBzfBjGRgH3EzTvwI0IV57HhlUjeyZc= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.207/go.mod h1:DaBrBCMgKGd3t7zg7z11jKm+0mVJiesr/IQCRG9qgP0= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 6ab1463a90d7c027d51647c593495ceec8332e3f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 4 Aug 2025 10:24:26 +0200 Subject: [PATCH 105/173] chore: go mod tidy --- router-tests/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index d47f42cb0d..a3c16da87e 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250718181713-66224598e91f + github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 From 957857b8713c41e069b4bc659c2a729182a033cd Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 4 Aug 2025 11:53:41 +0200 Subject: [PATCH 106/173] chore: add subscription_event_updater tests --- .../subscription_event_updater_test.go | 542 ++++++++++++++++++ 1 file changed, 542 insertions(+) create mode 100644 router/pkg/pubsub/datasource/subscription_event_updater_test.go diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go new file mode 100644 index 0000000000..ed56b20f7f --- /dev/null +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -0,0 +1,542 @@ +package datasource + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" +) + +// Test helper type for subscription event configuration +type testSubscriptionEventConfig struct { + providerID string + providerType ProviderType + fieldName string +} + +func (c *testSubscriptionEventConfig) ProviderID() string { + return c.providerID +} + +func (c *testSubscriptionEventConfig) ProviderType() ProviderType { + return c.providerType +} + +func (c *testSubscriptionEventConfig) RootFieldName() string { + return c.fieldName +} + +type receivedHooksArgs struct { + events []StreamEvent + cfg SubscriptionEventConfiguration +} + +func TestSubscriptionEventUpdater_Update_NoHooks(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data 1")}, + &testEvent{data: []byte("test data 2")}, + } + + // Expect calls to Update for each event + mockUpdater.On("Update", []byte("test data 1")).Return() + mockUpdater.On("Update", []byte("test data 2")).Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, // No hooks + } + + err := updater.Update(events) + + assert.NoError(t, err) +} + +func TestSubscriptionEventUpdater_Update_WithHooks_Success(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original data")}, + } + modifiedEvents := []StreamEvent{ + &testEvent{data: []byte("modified data")}, + } + + // Create wrapper function for the mock + receivedArgs := make(chan receivedHooksArgs, 1) + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs <- receivedHooksArgs{events: events, cfg: cfg} + return modifiedEvents, nil + } + + // Expect call to Update with modified data + mockUpdater.On("Update", []byte("modified data")).Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + }, + } + + err := updater.Update(originalEvents) + + select { + case receivedArgs := <-receivedArgs: + assert.Equal(t, originalEvents, receivedArgs.events) + assert.Equal(t, config, receivedArgs.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + assert.NoError(t, err) +} + +func TestSubscriptionEventUpdater_Update_WithHooks_Error(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + hookError := errors.New("hook processing error") + + // Define hook that returns an error + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, hookError + } + + // Should not call Update on eventUpdater since hook fails + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + }, + } + + err := updater.Update(events) + + assert.Error(t, err) + assert.Equal(t, hookError, err) + // Assert that Update was not called on the eventUpdater + mockUpdater.AssertNotCalled(t, "Update") +} + +func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + + // Chain of hooks that modify the data + receivedArgs1 := make(chan receivedHooksArgs, 1) + hook1 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs1 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("modified by hook1")}}, nil + } + + receivedArgs2 := make(chan receivedHooksArgs, 1) + hook2 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs2 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("modified by hook2")}}, nil + } + + // Expect call to Update with data modified by hook2 (last hook) + mockUpdater.On("Update", []byte("modified by hook2")).Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnStreamEvents: []OnStreamEventsFn{hook1, hook2}, + }, + } + + err := updater.Update(originalEvents) + + select { + case receivedArgs1 := <-receivedArgs1: + assert.Equal(t, originalEvents, receivedArgs1.events) + assert.Equal(t, config, receivedArgs1.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + select { + case receivedArgs2 := <-receivedArgs2: + assert.Equal(t, []StreamEvent{&testEvent{data: []byte("modified by hook1")}}, receivedArgs2.events) + assert.Equal(t, config, receivedArgs2.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + assert.NoError(t, err) +} + +func TestSubscriptionEventUpdater_Complete(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + + mockUpdater.On("Complete").Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, + } + + updater.Complete() +} + +func TestSubscriptionEventUpdater_Close(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + closeKind := resolve.SubscriptionCloseKindNormal + + mockUpdater.On("Close", closeKind).Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, + } + + updater.Close(closeKind) +} + +func TestSubscriptionEventUpdater_SetHooks(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return events, nil + } + + hooks := Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + } + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, + } + + updater.SetHooks(hooks) + + assert.Equal(t, hooks, updater.hooks) +} + +func TestNewSubscriptionEventUpdater(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return events, nil + } + + hooks := Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + } + + updater := NewSubscriptionEventUpdater(ctx, config, hooks, mockUpdater) + + assert.NotNil(t, updater) + + // Type assertion to access private fields for testing + var concreteUpdater *subscriptionEventUpdater + assert.IsType(t, concreteUpdater, updater) + concreteUpdater = updater.(*subscriptionEventUpdater) + assert.Equal(t, ctx, concreteUpdater.ctx) + assert.Equal(t, config, concreteUpdater.subscriptionEventConfiguration) + assert.Equal(t, hooks, concreteUpdater.hooks) + assert.Equal(t, mockUpdater, concreteUpdater.eventUpdater) +} + +func TestApplyStreamEventHooks_NoHooks(t *testing.T) { + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{}) + + assert.NoError(t, err) + assert.Equal(t, originalEvents, result) +} + +func TestApplyStreamEventHooks_SingleHook_Success(t *testing.T) { + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + modifiedEvents := []StreamEvent{ + &testEvent{data: []byte("modified")}, + } + + hook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return modifiedEvents, nil + } + + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook}) + + assert.NoError(t, err) + assert.Equal(t, modifiedEvents, result) +} + +func TestApplyStreamEventHooks_SingleHook_Error(t *testing.T) { + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + hookError := errors.New("hook processing failed") + + hook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, hookError + } + + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook}) + + assert.Error(t, err) + assert.Equal(t, hookError, err) + assert.Nil(t, result) +} + +func TestApplyStreamEventHooks_MultipleHooks_Success(t *testing.T) { + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + + receivedArgs1 := make(chan receivedHooksArgs, 1) + hook1 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs1 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("step1")}}, nil + } + receivedArgs2 := make(chan receivedHooksArgs, 1) + hook2 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs2 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("step2")}}, nil + } + receivedArgs3 := make(chan receivedHooksArgs, 1) + hook3 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs3 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("final")}}, nil + } + + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook1, hook2, hook3}) + + select { + case receivedArgs1 := <-receivedArgs1: + assert.Equal(t, originalEvents, receivedArgs1.events) + assert.Equal(t, config, receivedArgs1.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + select { + case receivedArgs2 := <-receivedArgs2: + assert.Equal(t, []StreamEvent{&testEvent{data: []byte("step1")}}, receivedArgs2.events) + assert.Equal(t, config, receivedArgs2.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + select { + case receivedArgs3 := <-receivedArgs3: + assert.Equal(t, []StreamEvent{&testEvent{data: []byte("step2")}}, receivedArgs3.events) + assert.Equal(t, config, receivedArgs3.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + assert.NoError(t, err) + assert.Len(t, result, 1) + assert.Equal(t, "final", string(result[0].GetData())) +} + +func TestApplyStreamEventHooks_MultipleHooks_MiddleHookError(t *testing.T) { + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + originalEvents := []StreamEvent{ + &testEvent{data: []byte("original")}, + } + middleHookError := errors.New("middle hook failed") + + receivedArgs1 := make(chan receivedHooksArgs, 1) + hook1 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs1 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("step1")}}, nil + } + receivedArgs2 := make(chan receivedHooksArgs, 1) + hook2 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs2 <- receivedHooksArgs{events: events, cfg: cfg} + return nil, middleHookError + } + receivedArgs3 := make(chan receivedHooksArgs, 1) + hook3 := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + receivedArgs3 <- receivedHooksArgs{events: events, cfg: cfg} + return []StreamEvent{&testEvent{data: []byte("final")}}, nil + } + + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook1, hook2, hook3}) + + assert.Error(t, err) + assert.Equal(t, middleHookError, err) + assert.Nil(t, result) + + select { + case receivedArgs1 := <-receivedArgs1: + assert.Equal(t, originalEvents, receivedArgs1.events) + assert.Equal(t, config, receivedArgs1.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + select { + case receivedArgs2 := <-receivedArgs2: + assert.Equal(t, []StreamEvent{&testEvent{data: []byte("step1")}}, receivedArgs2.events) + assert.Equal(t, config, receivedArgs2.cfg) + case <-time.After(1 * time.Second): + t.Fatal("timeout waiting for events") + } + + assert.Empty(t, receivedArgs3) +} + +// Test the updateEvents method indirectly through Update method +func TestSubscriptionEventUpdater_UpdateEvents_EmptyEvents(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{} // Empty events + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, // No hooks + } + + err := updater.Update(events) + + assert.NoError(t, err) + // No calls to Update should be made for empty events + mockUpdater.AssertNotCalled(t, "Update") +} + +func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { + testCases := []struct { + name string + closeKind resolve.SubscriptionCloseKind + }{ + {"Normal", resolve.SubscriptionCloseKindNormal}, + {"DownstreamServiceError", resolve.SubscriptionCloseKindDownstreamServiceError}, + {"GoingAway", resolve.SubscriptionCloseKindGoingAway}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + + mockUpdater.On("Close", tc.closeKind).Return() + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{}, + } + + updater.Close(tc.closeKind) + }) + } +} From 4d93bd13649e5956033ba2aa2f16daa05a3e716e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 4 Aug 2025 12:16:06 +0200 Subject: [PATCH 107/173] chore: change SubscriptionOnStartHandler to return only the error and change behaviour based on error type --- .../modules/start-subscription/module.go | 6 +- .../modules/start_subscription_test.go | 22 +++---- router/core/errors.go | 8 +-- router/core/graphql_handler.go | 16 ++--- router/core/router_config.go | 2 +- router/core/subscriptions_modules.go | 66 ++++++++++++------- 6 files changed, 71 insertions(+), 49 deletions(-) diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go index 60d4dc652c..6ece70d09c 100644 --- a/router-tests/modules/start-subscription/module.go +++ b/router-tests/modules/start-subscription/module.go @@ -10,7 +10,7 @@ const myModuleID = "startSubscriptionModule" type StartSubscriptionModule struct { Logger *zap.Logger - Callback func(ctx core.SubscriptionOnStartHookContext) (bool, error) + Callback func(ctx core.SubscriptionOnStartHookContext) error } func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { @@ -20,7 +20,7 @@ func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { return nil } -func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) (bool, error) { +func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { m.Logger.Info("SubscriptionOnStart Hook has been run") @@ -28,7 +28,7 @@ func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnSta return m.Callback(ctx) } - return false, nil + return nil } func (m *StartSubscriptionModule) Module() core.ModuleInfo { diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index e23c75ca5f..4d8609b6f1 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -90,12 +90,12 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + Callback: func(ctx core.SubscriptionOnStartHookContext) error { ctx.WriteEvent(&kafka.Event{ Key: []byte("1"), Data: []byte(`{"id": 1, "__typename": "Employee"}`), }) - return false, nil + return nil }, }, }, @@ -176,9 +176,9 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + Callback: func(ctx core.SubscriptionOnStartHookContext) error { callbackCalled <- true - return true, nil + return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "", true) }, }, }, @@ -255,15 +255,15 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + Callback: func(ctx core.SubscriptionOnStartHookContext) error { employeeId := ctx.RequestContext().Operation().Variables().GetInt64("employeeID") if employeeId != 1 { - return false, nil + return nil } ctx.WriteEvent(&kafka.Event{ Data: []byte(`{"id": 1, "__typename": "Employee"}`), }) - return false, nil + return nil }, }, }, @@ -359,8 +359,8 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { - return false, core.NewCustomModuleError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected)) + Callback: func(ctx core.SubscriptionOnStartHookContext) error { + return core.NewStreamHookError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected), false) }, }, }, @@ -502,11 +502,11 @@ func TestStartSubscriptionHook(t *testing.T) { Graph: config.Graph{}, Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ - Callback: func(ctx core.SubscriptionOnStartHookContext) (bool, error) { + Callback: func(ctx core.SubscriptionOnStartHookContext) error { ctx.WriteEvent(&core.EngineEvent{ Data: []byte(`{"data":{"countEmp":1000}}`), }) - return false, nil + return nil }, }, }, diff --git a/router/core/errors.go b/router/core/errors.go index 070463ca58..44e05f327b 100644 --- a/router/core/errors.go +++ b/router/core/errors.go @@ -35,7 +35,7 @@ const ( errorTypeInvalidWsSubprotocol errorTypeEDFSInvalidMessage errorTypeMergeResult - errorTypeCustomModuleError + errorTypeStreamHookError ) type ( @@ -90,9 +90,9 @@ func getErrorType(err error) errorType { if errors.As(err, &mergeResultErr) { return errorTypeMergeResult } - var customModuleErr *CustomModuleError - if errors.As(err, &customModuleErr) { - return errorTypeCustomModuleError + var streamHookErr *StreamHookError + if errors.As(err, &streamHookErr) { + return errorTypeStreamHookError } return errorTypeUnknown } diff --git a/router/core/graphql_handler.go b/router/core/graphql_handler.go index 921de2e5ed..f387d73e6c 100644 --- a/router/core/graphql_handler.go +++ b/router/core/graphql_handler.go @@ -400,21 +400,21 @@ func (h *GraphQLHandler) WriteError(ctx *resolve.Context, err error, res *resolv if isHttpResponseWriter { httpWriter.WriteHeader(http.StatusInternalServerError) } - case errorTypeCustomModuleError: - var customModuleErr *CustomModuleError - if !errors.As(err, &customModuleErr) { + case errorTypeStreamHookError: + var streamHookErr *StreamHookError + if !errors.As(err, &streamHookErr) { response.Errors[0].Message = "Internal server error" return } - response.Errors[0].Message = customModuleErr.Message() - if customModuleErr.Code() != "" || customModuleErr.StatusCode() != 0 { + response.Errors[0].Message = streamHookErr.Message() + if streamHookErr.Code() != "" || streamHookErr.StatusCode() != 0 { response.Errors[0].Extensions = &Extensions{ - Code: customModuleErr.Code(), - StatusCode: customModuleErr.StatusCode(), + Code: streamHookErr.Code(), + StatusCode: streamHookErr.StatusCode(), } } if isHttpResponseWriter { - httpWriter.WriteHeader(customModuleErr.StatusCode()) + httpWriter.WriteHeader(streamHookErr.StatusCode()) } } diff --git a/router/core/router_config.go b/router/core/router_config.go index 38ea0ac910..5de767d6ee 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -26,7 +26,7 @@ import ( ) type subscriptionHooks struct { - onStart []func(ctx SubscriptionOnStartHookContext) (bool, error) + onStart []func(ctx SubscriptionOnStartHookContext) error } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 18e1ace8ae..f91eba1892 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -1,44 +1,52 @@ package core import ( + "errors" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -// CustomModuleError is used to customize the error messages and the behavior -type CustomModuleError struct { - err error - message string - statusCode int - code string +// StreamHookError is used to customize the error messages and the behavior +type StreamHookError struct { + err error + message string + statusCode int + code string + closeConnection bool } -func (e *CustomModuleError) Error() string { +func (e *StreamHookError) Error() string { if e.err != nil { return e.err.Error() } return e.message } -func (e *CustomModuleError) Message() string { +func (e *StreamHookError) Message() string { return e.message } -func (e *CustomModuleError) StatusCode() int { +func (e *StreamHookError) StatusCode() int { return e.statusCode } -func (e *CustomModuleError) Code() string { +func (e *StreamHookError) Code() string { return e.code } -func NewCustomModuleError(err error, message string, statusCode int, code string) *CustomModuleError { - return &CustomModuleError{ - err: err, - message: message, - statusCode: statusCode, - code: code, +func (e *StreamHookError) CloseConnection() bool { + return e.closeConnection +} + +func NewStreamHookError(err error, message string, statusCode int, code string, closeConnection bool) *StreamHookError { + return &StreamHookError{ + err: err, + message: message, + statusCode: statusCode, + code: code, + closeConnection: closeConnection, } } @@ -98,13 +106,13 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() type SubscriptionOnStartHandler interface { // SubscriptionOnStart is called once at subscription start - // If the boolean is true, the subscription is closed. + // If the error is a StreamHookError and CloseConnection is true, the subscription is closed. // The error is propagated to the client. - SubscriptionOnStart(ctx SubscriptionOnStartHookContext) (bool, error) + SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } // NewPubSubSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a pubsub.SubscriptionOnStartFn -func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) datasource.SubscriptionOnStartFn { +func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) error) datasource.SubscriptionOnStartFn { if fn == nil { return nil } @@ -117,14 +125,21 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } - close, err := fn(hookCtx) + err := fn(hookCtx) + + // Check if the error is a StreamHookError and should close the connection + var streamHookErr *StreamHookError + close := false + if errors.As(err, &streamHookErr) { + close = streamHookErr.CloseConnection() + } return close, err } } // NewEngineSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.SubscriptionOnStartFn -func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) (bool, error)) graphql_datasource.SubscriptionOnStartFn { +func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) error) graphql_datasource.SubscriptionOnStartFn { if fn == nil { return nil } @@ -136,7 +151,14 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } - close, err := fn(hookCtx) + err := fn(hookCtx) + + // Check if the error is a StreamHookError and should close the connection + var streamHookErr *StreamHookError + close := false + if errors.As(err, &streamHookErr) { + close = streamHookErr.CloseConnection() + } return close, err } From e309be29ef24cb8cfcde8966b2def78d391ffe37 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 4 Aug 2025 14:44:13 +0200 Subject: [PATCH 108/173] chore: add what changes should be made to cosmo streams hooks with modules V1 --- adr/cosmo-streams-v1.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 44fddf0408..307913aa60 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -121,7 +121,12 @@ type StreamPublishEventHook interface { ## Backwards Compatibility The new hooks can be integrated in the router in a fully backwards compatible way. -When the new module system will be released, some changes will be needed. + +When the new module system will be released, the Cosmo Streams hooks: +- will be moved to the `core/hooks.go` file +- will be added to the `hookRegistry` +- will be initialized in the `coreModuleHooks.initCoreModuleHooks` + # Example Modules From cfc7a57dd5447a9bd25249396c7d60b5ecd9b0d1 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 5 Aug 2025 08:53:33 +0200 Subject: [PATCH 109/173] chore: add failing test --- .../modules/start-subscription/module.go | 16 +++- .../modules/start_subscription_test.go | 83 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/router-tests/modules/start-subscription/module.go b/router-tests/modules/start-subscription/module.go index 6ece70d09c..fd5a9e0088 100644 --- a/router-tests/modules/start-subscription/module.go +++ b/router-tests/modules/start-subscription/module.go @@ -1,6 +1,8 @@ package start_subscription import ( + "net/http" + "go.uber.org/zap" "github.com/wundergraph/cosmo/router/core" @@ -9,8 +11,9 @@ import ( const myModuleID = "startSubscriptionModule" type StartSubscriptionModule struct { - Logger *zap.Logger - Callback func(ctx core.SubscriptionOnStartHookContext) error + Logger *zap.Logger + Callback func(ctx core.SubscriptionOnStartHookContext) error + CallbackOnOriginResponse func(response *http.Response, ctx core.RequestContext) *http.Response } func (m *StartSubscriptionModule) Provision(ctx *core.ModuleContext) error { @@ -31,6 +34,14 @@ func (m *StartSubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnSta return nil } +func (m *StartSubscriptionModule) OnOriginResponse(response *http.Response, ctx core.RequestContext) *http.Response { + if m.CallbackOnOriginResponse != nil { + return m.CallbackOnOriginResponse(response, ctx) + } + + return response +} + func (m *StartSubscriptionModule) Module() core.ModuleInfo { return core.ModuleInfo{ // This is the ID of your module, it must be unique @@ -46,4 +57,5 @@ func (m *StartSubscriptionModule) Module() core.ModuleInfo { // Interface guard var ( _ core.SubscriptionOnStartHandler = (*StartSubscriptionModule)(nil) + _ core.EnginePostOriginHandler = (*StartSubscriptionModule)(nil) ) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 4d8609b6f1..ca8cf5d7ee 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -577,4 +577,87 @@ func TestStartSubscriptionHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) + + t.Run("Test StartSubscription hook is called, return StreamHookError with CloseConnection true, response on OnOriginResponse should still be set", func(t *testing.T) { + t.Parallel() + originResponseCalled := make(chan *http.Response, 1) + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "startSubscriptionModule": start_subscription.StartSubscriptionModule{ + Callback: func(ctx core.SubscriptionOnStartHookContext) error { + return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "", true) + }, + CallbackOnOriginResponse: func(response *http.Response, ctx core.RequestContext) *http.Response { + originResponseCalled <- response + return response + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&start_subscription.StartSubscriptionModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + var subscriptionCountEmp struct { + CountEmp int `graphql:"countEmp(max: $max, intervalMilliseconds: $interval)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + vars := map[string]interface{}{ + "max": 0, + "interval": 0, + } + + type subscriptionArgs struct { + dataValue []byte + errValue error + } + subscriptionOneArgsCh := make(chan subscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionCountEmp, vars, func(dataValue []byte, errValue error) error { + subscriptionOneArgsCh <- subscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + testenv.AwaitChannelWithT(t, time.Second*10, subscriptionOneArgsCh, func(t *testing.T, args subscriptionArgs) { + require.Error(t, args.errValue) + require.Empty(t, args.dataValue) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + select { + case response := <-originResponseCalled: + require.NotNil(t, response) + case <-time.After(time.Second * 10): + t.Fatal("origin response not called") + } + + requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) } From fca32f6d33acc06a425873e7218546f4fa9becfb Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 5 Aug 2025 18:43:55 +0200 Subject: [PATCH 110/173] chore: update engine --- router-tests/modules/start_subscription_test.go | 15 ++++++--------- router/go.mod | 2 +- router/go.sum | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index ca8cf5d7ee..4a91a80132 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -244,7 +244,10 @@ func TestStartSubscriptionHook(t *testing.T) { requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") assert.Len(t, requestLog.All(), 1) - require.Len(t, subscriptionArgsCh, 0) + require.Len(t, subscriptionArgsCh, 1) + subscriptionArgs := <-subscriptionArgsCh + require.Error(t, subscriptionArgs.errValue) + require.Empty(t, subscriptionArgs.dataValue) }) }) @@ -587,7 +590,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { - return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "", true) + return core.NewStreamHookError(errors.New("subscription closed"), "subscription closed", http.StatusOK, "NotFound", true) }, CallbackOnOriginResponse: func(response *http.Response, ctx core.RequestContext) *http.Response { originResponseCalled <- response @@ -644,17 +647,11 @@ func TestStartSubscriptionHook(t *testing.T) { require.Empty(t, args.dataValue) }) - require.NoError(t, client.Close()) testenv.AwaitChannelWithT(t, time.Second*10, clientRunCh, func(t *testing.T, err error) { require.NoError(t, err) }, "unable to close client before timeout") - select { - case response := <-originResponseCalled: - require.NotNil(t, response) - case <-time.After(time.Second * 10): - t.Fatal("origin response not called") - } + require.Empty(t, originResponseCalled) requestLog := xEnv.Observer().FilterMessage("SubscriptionOnStart Hook has been run") assert.Len(t, requestLog.All(), 1) diff --git a/router/go.mod b/router/go.mod index cd2466aa53..b4e5347c17 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 16b0129b27..7be78eeb22 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 61e0ddcbfa9dfc336ee1c85db3d219228f886a33 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 5 Aug 2025 18:46:58 +0200 Subject: [PATCH 111/173] chore: update router --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- router-tests/go.mod | 4 ++-- router-tests/go.sum | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 904b5bd2df..5e5f471cd0 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 + github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 06e1c65526..02ddf8c3b6 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 h1:+n2t7+lXsqR85wa7IANB6y8JdSoUOrzwSxFhJMrgVcQ= -github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555/go.mod h1:QapzpsmO9xKjIft1R3+tVl8dAUu0ZvDyOZnfwRmqRZc= +github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac h1:SWumdDA9Kx8cCttwtXuL5j04ElOoBgYz3Ywvp+VtfpY= +github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac/go.mod h1:SL/ZbCFXw+KjErn3SKJNQlvKYL7hXomBoHPyuLUixxQ= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/go.mod b/router-tests/go.mod index a3c16da87e..6120db2c59 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,8 +25,8 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250731082520-d014b3ab1555 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 + github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 36afa53a20..4d557fe267 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7 h1:eLp4+z2txqVvGJnjd3lQmfJR2PrNqGPWGV+1Ldgq7xE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250729124331-87cb3a8c31f7/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From cd9e4e4994fda1ac8a088130cc75487fee7ce4b8 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 6 Aug 2025 09:56:17 +0200 Subject: [PATCH 112/173] chore: update engine --- router/go.mod | 2 +- router/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/go.mod b/router/go.mod index b4e5347c17..fbc5b5f63f 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 7be78eeb22..9a485f9a1e 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 1c4a1a95729d67bc70de1c00526525b1743cda8b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 6 Aug 2025 09:57:56 +0200 Subject: [PATCH 113/173] chore: update router --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- router-tests/go.mod | 4 ++-- router-tests/go.sum | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 5e5f471cd0..3483103043 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac + github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 02ddf8c3b6..20bd3ed298 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac h1:SWumdDA9Kx8cCttwtXuL5j04ElOoBgYz3Ywvp+VtfpY= -github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac/go.mod h1:SL/ZbCFXw+KjErn3SKJNQlvKYL7hXomBoHPyuLUixxQ= +github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd h1:72U7uIdZ5uimYmPh7egT/84rOG8qR6sOdASeI6nuFo4= +github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd/go.mod h1:RBUiLGhXKdJMe9wQA8zDzED71ptNrpL2Wv2obmKfMco= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/go.mod b/router-tests/go.mod index 6120db2c59..8c4393e24d 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,8 +25,8 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250805164355-fca32f6d33ac - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 + github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 4d557fe267..6e37cd1eee 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29 h1:NWGN8OW4FjhCkHxYLvw59NE8SzZ0IjXWvE/B6ifn3hs= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250805163150-ea8ab759db29/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From d1f9af9dff23f152a26d91b2be4abb96f65b75f0 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 6 Aug 2025 12:45:20 +0200 Subject: [PATCH 114/173] chore: add tests --- .../pkg/pubsub/kafka/engine_datasource_test.go | 16 ++++++++++++++++ router/pkg/pubsub/nats/engine_datasource_test.go | 16 ++++++++++++++++ .../pkg/pubsub/redis/engine_datasource_test.go | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index 86406dfd85..cd7c382a92 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -67,6 +67,22 @@ func TestPublishData_MarshalJSONTemplate(t *testing.T) { } } +func TestPublishData_PublishEventConfiguration(t *testing.T) { + data := publishData{ + Provider: "test-provider", + Topic: "test-topic", + FieldName: "test-field", + } + + evtCfg := &PublishEventConfiguration{ + Provider: data.Provider, + Topic: data.Topic, + FieldName: data.FieldName, + } + + assert.Equal(t, evtCfg, data.PublishEventConfiguration()) +} + func TestKafkaPublishDataSource_Load(t *testing.T) { tests := []struct { name string diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 88d8b9a5d8..3e3f30d9c0 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -51,6 +51,22 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { } } +func TestPublishData_PublishEventConfiguration(t *testing.T) { + data := publishData{ + Provider: "test-provider", + Subject: "test-subject", + FieldName: "test-field", + } + + evtCfg := &PublishAndRequestEventConfiguration{ + Provider: data.Provider, + Subject: data.Subject, + FieldName: data.FieldName, + } + + assert.Equal(t, evtCfg, data.PublishEventConfiguration()) +} + func TestNatsPublishDataSource_Load(t *testing.T) { tests := []struct { name string diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 2b4a7c02f3..558a9a7d65 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -49,6 +49,22 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { } } +func TestPublishData_PublishEventConfiguration(t *testing.T) { + data := publishData{ + Provider: "test-provider", + Channel: "test-channel", + FieldName: "test-field", + } + + evtCfg := &PublishEventConfiguration{ + Provider: data.Provider, + Channel: data.Channel, + FieldName: data.FieldName, + } + + assert.Equal(t, evtCfg, data.PublishEventConfiguration()) +} + func TestRedisPublishDataSource_Load(t *testing.T) { tests := []struct { name string From 36dfd4f1d4dccfda7828b2d20ab1164ffd320933 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 7 Aug 2025 23:57:39 +0200 Subject: [PATCH 115/173] chore: improve error behaviour of updater --- router/core/subscriptions_modules.go | 8 +- router/pkg/pubsub/datasource/error.go | 4 + .../datasource/subscription_datasource.go | 10 +- .../subscription_datasource_test.go | 29 ++-- .../datasource/subscription_event_updater.go | 26 +++- .../subscription_event_updater_test.go | 143 +++++++++++++++++- router/pkg/pubsub/kafka/adapter.go | 9 +- .../pubsub/kafka/engine_datasource_factory.go | 4 +- router/pkg/pubsub/kafka/provider_builder.go | 1 + router/pkg/pubsub/nats/adapter.go | 34 ++++- .../pubsub/nats/engine_datasource_factory.go | 4 +- router/pkg/pubsub/nats/provider_builder.go | 1 + router/pkg/pubsub/redis/adapter.go | 9 +- .../pubsub/redis/engine_datasource_factory.go | 4 +- router/pkg/pubsub/redis/provider_builder.go | 1 + 15 files changed, 253 insertions(+), 34 deletions(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 771bc3a25d..a6e8d0fa13 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -37,10 +37,12 @@ func (e *StreamHookError) Code() string { return e.code } -func (e *StreamHookError) CloseConnection() bool { +func (e *StreamHookError) CloseSubscription() bool { return e.closeConnection } +var _ datasource.ErrorWithCloseSubscription = &StreamHookError{} + func NewStreamHookError(err error, message string, statusCode int, code string, closeConnection bool) *StreamHookError { return &StreamHookError{ err: err, @@ -145,7 +147,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext var streamHookErr *StreamHookError close := false if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseConnection() + close = streamHookErr.CloseSubscription() } return close, err @@ -171,7 +173,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext var streamHookErr *StreamHookError close := false if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseConnection() + close = streamHookErr.CloseSubscription() } return close, err diff --git a/router/pkg/pubsub/datasource/error.go b/router/pkg/pubsub/datasource/error.go index f09b271688..b6f4b449f9 100644 --- a/router/pkg/pubsub/datasource/error.go +++ b/router/pkg/pubsub/datasource/error.go @@ -15,3 +15,7 @@ func NewError(publicMsg string, cause error) *Error { Internal: cause, } } + +type ErrorWithCloseSubscription interface { + CloseSubscription() bool +} \ No newline at end of file diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index f60011aa0c..ffdd2b55aa 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -6,6 +6,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) type uniqueRequestIdFn func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error @@ -16,6 +17,7 @@ type PubSubSubscriptionDataSource[C SubscriptionEventConfiguration] struct { pubSub Adapter uniqueRequestID uniqueRequestIdFn hooks Hooks + logger *zap.Logger } func (s *PubSubSubscriptionDataSource[C]) SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) { @@ -39,7 +41,7 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return errors.New("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.hooks, updater)) + return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.hooks, updater, s.logger)) } func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { @@ -64,9 +66,13 @@ func (s *PubSubSubscriptionDataSource[C]) SetHooks(hooks Hooks) { var _ SubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) var _ resolve.HookableSubscriptionDataSource = (*PubSubSubscriptionDataSource[SubscriptionEventConfiguration])(nil) -func NewPubSubSubscriptionDataSource[C SubscriptionEventConfiguration](pubSub Adapter, uniqueRequestIdFn uniqueRequestIdFn) *PubSubSubscriptionDataSource[C] { +func NewPubSubSubscriptionDataSource[C SubscriptionEventConfiguration](pubSub Adapter, uniqueRequestIdFn uniqueRequestIdFn, logger *zap.Logger) *PubSubSubscriptionDataSource[C] { + if logger == nil { + logger = zap.NewNop() + } return &PubSubSubscriptionDataSource[C]{ pubSub: pubSub, uniqueRequestID: uniqueRequestIdFn, + logger: logger, } } diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index 3de44de3bd..2ff0f7a663 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) // testSubscriptionEventConfiguration implements SubscriptionEventConfiguration for testing @@ -36,7 +37,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_Success(t * return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -61,7 +62,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionEventConfiguration_InvalidJSON return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) invalidInput := []byte(`{"invalid": json}`) result, err := dataSource.SubscriptionEventConfiguration(invalidInput) @@ -75,7 +76,7 @@ func TestPubSubSubscriptionDataSource_UniqueRequestID_Success(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) ctx := &resolve.Context{} input := []byte(`{"test": "data"}`) @@ -92,7 +93,7 @@ func TestPubSubSubscriptionDataSource_UniqueRequestID_Error(t *testing.T) { return expectedError } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) ctx := &resolve.Context{} input := []byte(`{"test": "data"}`) @@ -109,7 +110,7 @@ func TestPubSubSubscriptionDataSource_Start_Success(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -134,7 +135,7 @@ func TestPubSubSubscriptionDataSource_Start_NoConfiguration(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) invalidInput := []byte(`{"invalid": json}`) ctx := resolve.NewContext(context.Background()) @@ -151,7 +152,7 @@ func TestPubSubSubscriptionDataSource_Start_SubscribeError(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -178,7 +179,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_Success(t *testing.T) return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) testConfig := testSubscriptionEventConfiguration{ Topic: "test-topic", @@ -200,7 +201,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) // Add subscription start hooks hook1Called := false @@ -242,7 +243,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsClose(t *te return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) // Add hook that returns close=true hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { @@ -273,7 +274,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) expectedError := errors.New("hook error") // Add hook that returns an error @@ -306,7 +307,7 @@ func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) // Initially should have no hooks assert.Len(t, dataSource.hooks.SubscriptionOnStart, 0) @@ -336,7 +337,7 @@ func TestNewPubSubSubscriptionDataSource(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) assert.NotNil(t, dataSource) assert.Equal(t, mockAdapter, dataSource.pubSub) @@ -350,7 +351,7 @@ func TestPubSubSubscriptionDataSource_InterfaceCompliance(t *testing.T) { return nil } - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) + dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn, zap.NewNop()) // Test that it implements SubscriptionDataSource interface var _ SubscriptionDataSource = dataSource diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 1e107d2357..b66c586930 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -4,6 +4,7 @@ import ( "context" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) // SubscriptionEventUpdater is a wrapper around the SubscriptionUpdater interface @@ -21,6 +22,7 @@ type subscriptionEventUpdater struct { ctx context.Context subscriptionEventConfiguration SubscriptionEventConfiguration hooks Hooks + logger *zap.Logger } func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { @@ -37,7 +39,24 @@ func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnStreamEvents) if err != nil { - return err + // Check if the error is a StreamHookError and should close the subscription + // We use type assertion to check for the CloseSubscription method without importing core + if hookErr, ok := err.(ErrorWithCloseSubscription); ok { + if hookErr.CloseSubscription() { + // If CloseSubscription is true, return the error to close the subscription + return err + } + } + // For all other errors, just log them and continue + if s.logger != nil { + s.logger.Error( + "An error occurred while processing stream events hooks", + zap.Error(err), + zap.String("provider_type", string(s.subscriptionEventConfiguration.ProviderType())), + zap.String("provider_id", s.subscriptionEventConfiguration.ProviderID()), + zap.String("field_name", s.subscriptionEventConfiguration.RootFieldName()), + ) + } } s.updateEvents(processedEvents) @@ -79,11 +98,14 @@ func NewSubscriptionEventUpdater( ctx context.Context, cfg SubscriptionEventConfiguration, hooks Hooks, - eventUpdater resolve.SubscriptionUpdater) SubscriptionEventUpdater { + eventUpdater resolve.SubscriptionUpdater, + logger *zap.Logger, +) SubscriptionEventUpdater { return &subscriptionEventUpdater{ ctx: ctx, subscriptionEventConfiguration: cfg, hooks: hooks, eventUpdater: eventUpdater, + logger: logger, } } diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go index ed56b20f7f..2ca7407e91 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater_test.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) // Test helper type for subscription event configuration @@ -140,8 +142,8 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error(t *testing.T) { err := updater.Update(events) - assert.Error(t, err) - assert.Equal(t, hookError, err) + // With the new behavior, errors are logged and nil is returned + assert.NoError(t, err) // Assert that Update was not called on the eventUpdater mockUpdater.AssertNotCalled(t, "Update") } @@ -293,7 +295,7 @@ func TestNewSubscriptionEventUpdater(t *testing.T) { OnStreamEvents: []OnStreamEventsFn{testHook}, } - updater := NewSubscriptionEventUpdater(ctx, config, hooks, mockUpdater) + updater := NewSubscriptionEventUpdater(ctx, config, hooks, mockUpdater, zap.NewNop()) assert.NotNil(t, updater) @@ -540,3 +542,138 @@ func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { }) } } + +func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + + // Create a mock StreamHookError with CloseSubscription=true + mockHookError := &mockStreamHookError{ + closeSubscription: true, + message: "subscription should close", + } + + // Define hook that returns a StreamHookError with CloseSubscription=true + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, mockHookError + } + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + }, + } + + err := updater.Update(events) + + // Should return the error when CloseSubscription is true + assert.Error(t, err) + assert.Equal(t, mockHookError, err) + // Assert that Update was not called on the eventUpdater + mockUpdater.AssertNotCalled(t, "Update") +} + +func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + + // Create a mock StreamHookError with CloseSubscription=false + mockHookError := &mockStreamHookError{ + closeSubscription: false, + message: "subscription should not close", + } + + // Define hook that returns a StreamHookError with CloseSubscription=false + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, mockHookError + } + + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + ctx: ctx, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + }, + } + + err := updater.Update(events) + + // Should return nil when CloseSubscription is false (error is logged) + assert.NoError(t, err) + // Assert that Update was not called on the eventUpdater + mockUpdater.AssertNotCalled(t, "Update") +} + +func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + ctx := context.Background() + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } + hookError := errors.New("hook processing error") + + // Define hook that returns an error + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return nil, hookError + } + + zCore, logObserver := observer.New(zap.InfoLevel) + logger := zap.New(zCore) + + // Test with a real zap logger to verify error logging behavior + // The logger.Error() call should be executed when an error occurs + updater := NewSubscriptionEventUpdater(ctx, config, Hooks{ + OnStreamEvents: []OnStreamEventsFn{testHook}, + }, mockUpdater, logger) + + err := updater.Update(events) + + // Should return nil when error is logged + assert.NoError(t, err) + // Assert that Update was not called on the eventUpdater + mockUpdater.AssertNotCalled(t, "Update") + + msgs := logObserver.FilterMessageSnippet("An error occurred while processing stream events hooks").TakeAll() + assert.Equal(t, 1, len(msgs)) +} + +// mockStreamHookError implements the CloseSubscription() method for testing +type mockStreamHookError struct { + closeSubscription bool + message string +} + +func (e *mockStreamHookError) Error() string { + return e.message +} + +func (e *mockStreamHookError) CloseSubscription() bool { + return e.closeSubscription +} + + diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index b23b3d65f3..a5b63d7b26 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -144,8 +144,13 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri if errors.Is(err, errClientClosed) || errors.Is(err, context.Canceled) { log.Debug("poller canceled", zap.Error(err)) } else { - log.Error("poller error", zap.Error(err)) - + log.Error( + "poller error", + zap.Error(err), + zap.String("provider_id", conf.ProviderID()), + zap.String("provider_type", string(conf.ProviderType())), + zap.String("field_name", conf.RootFieldName()), + ) } return } diff --git a/router/pkg/pubsub/kafka/engine_datasource_factory.go b/router/pkg/pubsub/kafka/engine_datasource_factory.go index c26f67e92a..d89eb408b0 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_factory.go +++ b/router/pkg/pubsub/kafka/engine_datasource_factory.go @@ -8,6 +8,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) type EventType int @@ -22,6 +23,7 @@ type EngineDataSourceFactory struct { eventType EventType topics []string providerId string + logger *zap.Logger KafkaAdapter datasource.Adapter } @@ -81,7 +83,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.Su _, err = xxh.Write(val) return err - }), nil + }, c.logger), nil } func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { diff --git a/router/pkg/pubsub/kafka/provider_builder.go b/router/pkg/pubsub/kafka/provider_builder.go index 1a84d8be51..e798694bb4 100644 --- a/router/pkg/pubsub/kafka/provider_builder.go +++ b/router/pkg/pubsub/kafka/provider_builder.go @@ -52,6 +52,7 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.KafkaEventCo topics: data.GetTopics(), providerId: providerId, KafkaAdapter: provider, + logger: p.logger, }, nil } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 75e4133c18..eb12728c9f 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -138,7 +138,14 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip }}) if updateErr != nil { // If the update fails, we do not acknowledge the message - log.Error("error updating subscription", zap.Error(updateErr), zap.String("message_subject", msg.Subject())) + log.Error( + "error updating subscription, stopping subscription", + zap.Error(updateErr), + zap.String("message_subject", msg.Subject()), + zap.String("provider_id", subConf.ProviderID()), + zap.String("provider_type", string(subConf.ProviderType())), + zap.String("field_name", subConf.RootFieldName()), + ) return } @@ -183,7 +190,14 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip }}) if updateErr != nil { // If the update fails, we log the error and unsubscribe from all subscriptions - log.Error("error updating subscription", zap.Error(updateErr), zap.String("message_subject", msg.Subject)) + log.Error( + "error updating subscription, stopping subscription", + zap.Error(updateErr), + zap.String("message_subject", msg.Subject), + zap.String("provider_id", subConf.ProviderID()), + zap.String("provider_type", string(subConf.ProviderType())), + zap.String("field_name", subConf.RootFieldName()), + ) for _, subscription := range subscriptions { if err := subscription.Unsubscribe(); err != nil { log.Error("unsubscribing from NATS subject after an error on updating subscription", @@ -250,7 +264,13 @@ func (p *ProviderAdapter) Publish(_ context.Context, conf datasource.PublishEven err := p.client.Publish(pubConf.Subject, natsEvent.Data) if err != nil { - log.Error("publish error", zap.Error(err)) + log.Error( + "publish error", + zap.Error(err), + zap.String("provider_id", pubConf.ProviderID()), + zap.String("provider_type", string(pubConf.ProviderType())), + zap.String("field_name", pubConf.RootFieldName()), + ) return datasource.NewError(fmt.Sprintf("error publishing to NATS subject %s", pubConf.Subject), err) } } @@ -283,7 +303,13 @@ func (p *ProviderAdapter) Request(ctx context.Context, cfg datasource.PublishEve msg, err := p.client.RequestWithContext(ctx, reqConf.Subject, natsEvent.Data) if err != nil { - log.Error("request error", zap.Error(err)) + log.Error( + "request error", + zap.Error(err), + zap.String("provider_id", reqConf.ProviderID()), + zap.String("provider_type", string(reqConf.ProviderType())), + zap.String("field_name", reqConf.RootFieldName()), + ) return datasource.NewError(fmt.Sprintf("error requesting from NATS subject %s", reqConf.Subject), err) } diff --git a/router/pkg/pubsub/nats/engine_datasource_factory.go b/router/pkg/pubsub/nats/engine_datasource_factory.go index 3f3c8ebc9b..d88d25b868 100644 --- a/router/pkg/pubsub/nats/engine_datasource_factory.go +++ b/router/pkg/pubsub/nats/engine_datasource_factory.go @@ -9,6 +9,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) type EventType int @@ -26,6 +27,7 @@ type EngineDataSourceFactory struct { eventType EventType subjects []string providerId string + logger *zap.Logger withStreamConfiguration bool consumerName string @@ -94,7 +96,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.Su _, err = xxh.Write(val) return err - }), nil + }, c.logger), nil } func (c *EngineDataSourceFactory) ResolveDataSourceSubscriptionInput() (string, error) { diff --git a/router/pkg/pubsub/nats/provider_builder.go b/router/pkg/pubsub/nats/provider_builder.go index 91eed2e839..18e6f8f094 100644 --- a/router/pkg/pubsub/nats/provider_builder.go +++ b/router/pkg/pubsub/nats/provider_builder.go @@ -51,6 +51,7 @@ func (p *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.NatsEventCon subjects: data.GetSubjects(), providerId: providerId, withStreamConfiguration: data.GetStreamConfiguration() != nil, + logger: p.logger, } if data.GetStreamConfiguration() != nil { diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 67a3f1542b..f05736351a 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -118,7 +118,14 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri Data: []byte(msg.Payload), }}) if updateErr != nil { - log.Error("error updating subscription, stopping subscription", zap.Error(updateErr), zap.String("message_channel", msg.Channel)) + log.Error( + "error updating subscription, stopping subscription", + zap.Error(updateErr), + zap.String("message_channel", msg.Channel), + zap.String("provider_id", conf.ProviderID()), + zap.String("provider_type", string(conf.ProviderType())), + zap.String("field_name", conf.RootFieldName()), + ) // If the error is not recoverable, we stop the subscription cleanup() return diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index dd76bdc7ab..0d25716f3e 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -9,6 +9,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) type EventType int @@ -26,6 +27,7 @@ type EngineDataSourceFactory struct { eventType EventType channels []string providerId string + logger *zap.Logger } func (c *EngineDataSourceFactory) GetFieldName() string { @@ -92,7 +94,7 @@ func (c *EngineDataSourceFactory) ResolveDataSourceSubscription() (datasource.Su _, err = xxh.Write(val) return err - }), nil + }, c.logger), nil } // ResolveDataSourceSubscriptionInput builds the input for the subscription data source diff --git a/router/pkg/pubsub/redis/provider_builder.go b/router/pkg/pubsub/redis/provider_builder.go index 6e211fec33..f0a0c2eb9c 100644 --- a/router/pkg/pubsub/redis/provider_builder.go +++ b/router/pkg/pubsub/redis/provider_builder.go @@ -64,6 +64,7 @@ func (b *ProviderBuilder) BuildEngineDataSourceFactory(data *nodev1.RedisEventCo channels: data.GetChannels(), providerId: providerId, RedisAdapter: provider, + logger: b.logger, }, nil } From ceac25583049c027f289f4508d6477355298fc65 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 09:35:31 +0200 Subject: [PATCH 116/173] chore: rename CloseConnection to CloseSubscription --- router/core/subscriptions_modules.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index f91eba1892..8934c463df 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -14,7 +14,7 @@ type StreamHookError struct { message string statusCode int code string - closeConnection bool + closeSubscription bool } func (e *StreamHookError) Error() string { @@ -36,17 +36,17 @@ func (e *StreamHookError) Code() string { return e.code } -func (e *StreamHookError) CloseConnection() bool { - return e.closeConnection +func (e *StreamHookError) CloseSubscription() bool { + return e.closeSubscription } -func NewStreamHookError(err error, message string, statusCode int, code string, closeConnection bool) *StreamHookError { +func NewStreamHookError(err error, message string, statusCode int, code string, closeSubscription bool) *StreamHookError { return &StreamHookError{ err: err, message: message, statusCode: statusCode, code: code, - closeConnection: closeConnection, + closeSubscription: closeSubscription, } } @@ -106,7 +106,7 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() type SubscriptionOnStartHandler interface { // SubscriptionOnStart is called once at subscription start - // If the error is a StreamHookError and CloseConnection is true, the subscription is closed. + // If the error is a StreamHookError and CloseSubscription is true, the subscription is closed. // The error is propagated to the client. SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } @@ -131,7 +131,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext var streamHookErr *StreamHookError close := false if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseConnection() + close = streamHookErr.CloseSubscription() } return close, err @@ -157,7 +157,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext var streamHookErr *StreamHookError close := false if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseConnection() + close = streamHookErr.CloseSubscription() } return close, err From 266c096adcf427de41c39e78042c0fc02049779f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 12:54:42 +0200 Subject: [PATCH 117/173] chore: improved error on publish behaviour --- router-tests/events/kafka_events_test.go | 28 ++-- router-tests/events/redis_events_test.go | 98 +++---------- router-tests/events/utils.go | 54 ++++++- router-tests/modules/stream_batch_test.go | 4 +- router-tests/modules/stream_publish_test.go | 136 +++++++++++++++++- .../pkg/pubsub/datasource/pubsubprovider.go | 21 ++- router/pkg/pubsub/kafka/engine_datasource.go | 15 +- 7 files changed, 254 insertions(+), 102 deletions(-) diff --git a/router-tests/events/kafka_events_test.go b/router-tests/events/kafka_events_test.go index ffd08736c4..0290a76578 100644 --- a/router-tests/events/kafka_events_test.go +++ b/router-tests/events/kafka_events_test.go @@ -73,7 +73,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -129,7 +129,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -203,7 +203,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -276,7 +276,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -365,7 +365,7 @@ func TestKafkaEvents(t *testing.T) { engineExecutionConfiguration.WebSocketClientReadTimeout = time.Millisecond * 100 }, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -430,7 +430,7 @@ func TestKafkaEvents(t *testing.T) { core.WithMultipartHeartbeatInterval(multipartHeartbeatInterval), }, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -496,7 +496,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -561,7 +561,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) subscribePayload := []byte(`{"query":"subscription { employeeUpdatedMyKafka(employeeID: 1) { id details { forename surname } }}"}`) @@ -671,7 +671,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -738,7 +738,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -805,7 +805,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { @@ -860,7 +860,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -931,7 +931,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) // Send a mutation to trigger the first subscription resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ @@ -979,7 +979,7 @@ func TestKafkaEvents(t *testing.T) { RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) + events.KafkaEnsureTopicExists(t, xEnv, KafkaWaitTimeout, topics...) type subscriptionPayload struct { Data struct { diff --git a/router-tests/events/redis_events_test.go b/router-tests/events/redis_events_test.go index 407e9a9348..305d846e57 100644 --- a/router-tests/events/redis_events_test.go +++ b/router-tests/events/redis_events_test.go @@ -3,20 +3,18 @@ package events_test import ( "bufio" "bytes" - "context" "encoding/json" "fmt" "net/http" - "net/url" "testing" "time" - "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/wundergraph/cosmo/router/core" "github.com/hasura/go-graphql-client" "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router-tests/events" "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/pkg/config" ) @@ -104,7 +102,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // process the message select { @@ -170,7 +168,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce an empty message - produceRedisMessage(t, xEnv, topics[0], ``) + events.ProduceRedisMessage(t, xEnv, topics[0], ``) // process the message select { case subscriptionArgs := <-subscriptionArgsCh: @@ -181,7 +179,7 @@ func TestRedisEvents(t *testing.T) { t.Fatal("timeout waiting for first message error") } - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Correct message select { case subscriptionArgs := <-subscriptionArgsCh: require.NoError(t, subscriptionArgs.errValue) @@ -191,7 +189,7 @@ func TestRedisEvents(t *testing.T) { } // Missing entity = Resolver error - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) select { case subscriptionArgs := <-subscriptionArgsCh: var gqlErr graphql.Errors @@ -202,7 +200,7 @@ func TestRedisEvents(t *testing.T) { } // Correct message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) select { case subscriptionArgs := <-subscriptionArgsCh: require.NoError(t, subscriptionArgs.errValue) @@ -273,7 +271,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(2, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message from the first subscription select { @@ -354,7 +352,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(2, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message from the first subscription select { @@ -375,7 +373,7 @@ func TestRedisEvents(t *testing.T) { } // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 2,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 2,"update":{"name":"foo"}}`) // read the message from the first subscription select { @@ -451,7 +449,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message from the subscription select { @@ -509,12 +507,12 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message from the subscription assertRedisMultipartValueEventually(t, reader, "{\"payload\":{\"data\":{\"employeeUpdates\":{\"id\":1,\"details\":{\"forename\":\"Jens\",\"surname\":\"Neuse\"}}}}}") // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message from the subscription assertRedisMultipartValueEventually(t, reader, "{\"payload\":{\"data\":{\"employeeUpdates\":{\"id\":1,\"details\":{\"forename\":\"Jens\",\"surname\":\"Neuse\"}}}}}") }) @@ -590,7 +588,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message so that the subscription is triggered - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // get the client response var clientRet struct { @@ -663,7 +661,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message so that the subscription is triggered - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // get the client response var clientRet struct { @@ -792,7 +790,7 @@ func TestRedisEvents(t *testing.T) { // Events 1, 3, 4, 7, and 11 should be included for i := MsgCount; i > 0; i-- { - produceRedisMessage(t, xEnv, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) + events.ProduceRedisMessage(t, xEnv, topics[0], fmt.Sprintf(`{"__typename":"Employee","id":%d}`, i)) if i == 11 || i == 7 || i == 4 || i == 3 || i == 1 { gErr := conn.ReadJSON(&msg) @@ -853,7 +851,7 @@ func TestRedisEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce an invalid message - produceRedisMessage(t, xEnv, topics[0], `{asas`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{asas`) // get the client response select { case args := <-subscriptionOneArgsCh: @@ -865,7 +863,7 @@ func TestRedisEvents(t *testing.T) { } // produce a correct message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id":1}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id":1}`) // get the client response select { case args := <-subscriptionOneArgsCh: @@ -876,7 +874,7 @@ func TestRedisEvents(t *testing.T) { } // produce a message with a missing entity - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","update":{"name":"foo"}}`) // get the client response select { case args := <-subscriptionOneArgsCh: @@ -888,7 +886,7 @@ func TestRedisEvents(t *testing.T) { } // produce a correct message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // get the client response select { case args := <-subscriptionOneArgsCh: @@ -920,7 +918,7 @@ func TestRedisEvents(t *testing.T) { NoRetryClient: true, }, func(t *testing.T, xEnv *testenv.Environment) { // start reading the messages from the channel - msgCh, err := readRedisMessages(t, xEnv, channels[0]) + msgCh, err := events.ReadRedisMessages(t, xEnv, channels[0]) require.NoError(t, err) // send a mutation to trigger the first subscription @@ -991,7 +989,7 @@ func TestRedisClusterEvents(t *testing.T) { xEnv.WaitForSubscriptionCount(1, RedisWaitTimeout) // produce a message - produceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + events.ProduceRedisMessage(t, xEnv, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // read the message select { @@ -1026,7 +1024,7 @@ func TestRedisClusterEvents(t *testing.T) { NoRetryClient: true, }, func(t *testing.T, xEnv *testenv.Environment) { // start reading the messages from the channel - msgCh, err := readRedisMessages(t, xEnv, channels[0]) + msgCh, err := events.ReadRedisMessages(t, xEnv, channels[0]) require.NoError(t, err) // send a mutation to produce a message @@ -1045,54 +1043,4 @@ func TestRedisClusterEvents(t *testing.T) { }) }) -} - -func produceRedisMessage(t *testing.T, xEnv *testenv.Environment, topicName string, message string) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - parsedURL, err := url.Parse(xEnv.RedisHosts[0]) - if err != nil { - t.Fatalf("Failed to parse Redis URL: %v", err) - } - var redisConn redis.UniversalClient - if !xEnv.RedisWithClusterMode { - redisConn = redis.NewClient(&redis.Options{ - Addr: parsedURL.Host, - }) - } else { - redisConn = redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{parsedURL.Host}, - }) - } - - intCmd := redisConn.Publish(ctx, xEnv.GetPubSubName(topicName), message) - require.NoError(t, intCmd.Err()) -} - -func readRedisMessages(t *testing.T, xEnv *testenv.Environment, channelName string) (<-chan *redis.Message, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - parsedURL, err := url.Parse(xEnv.RedisHosts[0]) - if err != nil { - return nil, err - } - var redisConn redis.UniversalClient - if !xEnv.RedisWithClusterMode { - redisConn = redis.NewClient(&redis.Options{ - Addr: parsedURL.Host, - }) - } else { - redisConn = redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{parsedURL.Host}, - }) - } - sub := redisConn.Subscribe(ctx, xEnv.GetPubSubName(channelName)) - t.Cleanup(func() { - sub.Close() - redisConn.Close() - }) - - return sub.Channel(), nil -} +} \ No newline at end of file diff --git a/router-tests/events/utils.go b/router-tests/events/utils.go index 406df3e560..03bac2d7d3 100644 --- a/router-tests/events/utils.go +++ b/router-tests/events/utils.go @@ -2,15 +2,17 @@ package events import ( "context" + "net/url" "testing" "time" + "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" "github.com/wundergraph/cosmo/router-tests/testenv" ) -func EnsureTopicExists(t *testing.T, xEnv *testenv.Environment, timeout time.Duration, topics ...string) { +func KafkaEnsureTopicExists(t *testing.T, xEnv *testenv.Environment, timeout time.Duration, topics ...string) { // Delete topic for idempotency deleteCtx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -67,3 +69,53 @@ func ReadKafkaMessages(xEnv *testenv.Environment, timeout time.Duration, topicNa return fetchs.Records(), nil } + +func ProduceRedisMessage(t *testing.T, xEnv *testenv.Environment, topicName string, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + parsedURL, err := url.Parse(xEnv.RedisHosts[0]) + if err != nil { + t.Fatalf("Failed to parse Redis URL: %v", err) + } + var redisConn redis.UniversalClient + if !xEnv.RedisWithClusterMode { + redisConn = redis.NewClient(&redis.Options{ + Addr: parsedURL.Host, + }) + } else { + redisConn = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{parsedURL.Host}, + }) + } + + intCmd := redisConn.Publish(ctx, xEnv.GetPubSubName(topicName), message) + require.NoError(t, intCmd.Err()) +} + +func ReadRedisMessages(t *testing.T, xEnv *testenv.Environment, channelName string) (<-chan *redis.Message, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + parsedURL, err := url.Parse(xEnv.RedisHosts[0]) + if err != nil { + return nil, err + } + var redisConn redis.UniversalClient + if !xEnv.RedisWithClusterMode { + redisConn = redis.NewClient(&redis.Options{ + Addr: parsedURL.Host, + }) + } else { + redisConn = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{parsedURL.Host}, + }) + } + sub := redisConn.Subscribe(ctx, xEnv.GetPubSubName(channelName)) + t.Cleanup(func() { + sub.Close() + redisConn.Close() + }) + + return sub.Channel(), nil +} \ No newline at end of file diff --git a/router-tests/modules/stream_batch_test.go b/router-tests/modules/stream_batch_test.go index 99ac1a1711..deeda4fb18 100644 --- a/router-tests/modules/stream_batch_test.go +++ b/router-tests/modules/stream_batch_test.go @@ -51,7 +51,7 @@ func TestBatchHook(t *testing.T) { }, }, func(t *testing.T, xEnv *testenv.Environment) { topics := []string{"employeeUpdated"} - events.EnsureTopicExists(t, xEnv, time.Second, topics...) + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { @@ -136,7 +136,7 @@ func TestBatchHook(t *testing.T) { }, }, func(t *testing.T, xEnv *testenv.Environment) { topics := []string{"employeeUpdated"} - events.EnsureTopicExists(t, xEnv, time.Second, topics...) + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) var subscriptionOne struct { employeeUpdatedMyKafka struct { diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index faa6d9856b..096a4bda96 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -1,6 +1,8 @@ package module_test import ( + "encoding/json" + "errors" "testing" "time" @@ -86,7 +88,7 @@ func TestPublishHook(t *testing.T) { LogLevel: zapcore.InfoLevel, }, }, func(t *testing.T, xEnv *testenv.Environment) { - events.EnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") + events.KafkaEnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, }) @@ -103,4 +105,136 @@ func TestPublishHook(t *testing.T) { require.Equal(t, []byte("test"), header.Value) }) }) + + t.Run("Test kafka publish error is returned and messages sent", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR", false) + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + events.KafkaEnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, + }) + require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": false}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) + require.NoError(t, err) + require.Len(t, records, 1) + }) + }) + + t.Run("Test nats publish error is returned and messages sent", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR", false) + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsNatsJSONTemplate, + EnableNats: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + firstSub, err := xEnv.NatsConnectionDefault.SubscribeSync(xEnv.GetPubSubName("employeeUpdatedMyNats.3")) + require.NoError(t, err) + t.Cleanup(func() { + _ = firstSub.Unsubscribe() + }) + require.NoError(t, xEnv.NatsConnectionDefault.Flush()) + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation UpdateEmployeeNats($update: UpdateEmployeeInput!) { + updateEmployeeMyNats(id: 3, update: $update) {success} + }`, + Variables: json.RawMessage(`{"update":{"name":"Stefan Avramovic","email":"avramovic@wundergraph.com"}}`), + }) + assert.JSONEq(t, `{"data": {"updateEmployeeMyNats": {"success": false}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + msgOne, err := firstSub.NextMsg(5 * time.Second) + require.NoError(t, err) + require.Equal(t, xEnv.GetPubSubName("employeeUpdatedMyNats.3"), msgOne.Subject) + require.Equal(t, `{"id":3,"update":{"name":"Stefan Avramovic","email":"avramovic@wundergraph.com"}}`, string(msgOne.Data)) + require.NoError(t, err) + }) + }) + + t.Run("Test redis publish error is returned and messages sent", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR", false) + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsRedisJSONTemplate, + EnableRedis: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, + }) + require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": false}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) + require.NoError(t, err) + require.Len(t, records, 1) + }) + }) } diff --git a/router/pkg/pubsub/datasource/pubsubprovider.go b/router/pkg/pubsub/datasource/pubsubprovider.go index 2d9514d5e8..e234ebfb73 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider.go +++ b/router/pkg/pubsub/datasource/pubsubprovider.go @@ -22,7 +22,7 @@ func applyPublishEventHooks(ctx context.Context, cfg PublishEventConfiguration, var err error currentEvents, err = hook(ctx, cfg, currentEvents) if err != nil { - return nil, err + return currentEvents, err } } return currentEvents, nil @@ -59,12 +59,23 @@ func (p *PubSubProvider) Publish(ctx context.Context, cfg PublishEventConfigurat return p.Adapter.Publish(ctx, cfg, events) } - processedEvents, err := applyPublishEventHooks(ctx, cfg, events, p.hooks.OnPublishEvents) - if err != nil { - return err + processedEvents, hooksErr := applyPublishEventHooks(ctx, cfg, events, p.hooks.OnPublishEvents) + if hooksErr != nil { + p.Logger.Error( + "error applying publish event hooks", + zap.Error(hooksErr), + zap.String("provider_id", cfg.ProviderID()), + zap.String("provider_type_id", string(cfg.ProviderType())), + zap.String("field_name", cfg.RootFieldName()), + ) + } + + errPublish := p.Adapter.Publish(ctx, cfg, processedEvents) + if errPublish != nil { + return errPublish } - return p.Adapter.Publish(ctx, cfg, processedEvents) + return hooksErr } func (p *PubSubProvider) SetHooks(hooks Hooks) { diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 92c6505e31..3c2bf97d36 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -162,11 +162,18 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B } if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { - _, err = io.WriteString(out, `{"success": false}`) - return err + _, errWrite := io.WriteString(out, `{"success": false}`) + if errWrite != nil { + return errWrite + } + // it will not be returned but only logged to avoid a "unable to fetch from subgraph" error + return nil } - _, err := io.WriteString(out, `{"success": true}`) - return err + _, errWrite := io.WriteString(out, `{"success": true}`) + if errWrite != nil { + return errWrite + } + return nil } func (s *PublishDataSource) LoadWithFiles(ctx context.Context, input []byte, files []*httpclient.FileUpload, out *bytes.Buffer) (err error) { From 4d39d4c4774146e07cd954735acd11fde22061e4 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 13:04:01 +0200 Subject: [PATCH 118/173] chore: fix failing test with new logic --- router/pkg/pubsub/datasource/pubsubprovider_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index f2618b0d06..51c7b239f3 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -239,19 +239,20 @@ func TestProvider_Publish_WithHooks_HookError(t *testing.T) { return nil, hookError } - // Should not call Publish on adapter since hook fails + mockAdapter.On("Publish", mock.Anything, config, []StreamEvent(nil)).Return(nil) + + // Should call Publish on adapter also if hook fails provider := PubSubProvider{ Adapter: mockAdapter, hooks: Hooks{ OnPublishEvents: []OnPublishEventsFn{testHook}, }, + Logger: zap.NewNop(), } err := provider.Publish(context.Background(), config, events) assert.Error(t, err) assert.Equal(t, hookError, err) - // Assert that Publish was not called on the adapter - mockAdapter.AssertNotCalled(t, "Publish") } func TestProvider_Publish_WithHooks_AdapterError(t *testing.T) { From 79595ea088802c691257f334e8a2b616343e66fc Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 15:55:22 +0200 Subject: [PATCH 119/173] chore: correctly test redis --- router-tests/modules/stream_publish_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 096a4bda96..0e36ae42d8 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -224,16 +224,17 @@ func TestPublishHook(t *testing.T) { LogLevel: zapcore.InfoLevel, }, }, func(t *testing.T, xEnv *testenv.Environment) { + records, err := events.ReadRedisMessages(t, xEnv, "employeeUpdatedMyRedis") + require.NoError(t, err) + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ - Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, + Query: `mutation { updateEmployeeMyRedis(id: 3, update: {name: "name test"}) { success } }`, }) - require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": false}}}`, resOne.Body) + require.JSONEq(t, `{"data": {"updateEmployeeMyRedis": {"success": false}}}`, resOne.Body) requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") assert.Len(t, requestLog.All(), 1) - - records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) - require.NoError(t, err) + require.Len(t, records, 1) }) }) From 2343e0307277340427f55a60e1393e150cda6831 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 16:14:57 +0200 Subject: [PATCH 120/173] chore: improved Publish method --- router-tests/modules/stream_publish_test.go | 11 ++++++++++- router/pkg/pubsub/kafka/engine_datasource.go | 9 +++------ router/pkg/pubsub/nats/engine_datasource.go | 5 +++-- router/pkg/pubsub/redis/engine_datasource.go | 5 +++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 0e36ae42d8..3d46e309e9 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -141,6 +141,9 @@ func TestPublishHook(t *testing.T) { requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") assert.Len(t, requestLog.All(), 1) + requestLog2 := xEnv.Observer().FilterMessage("error applying publish event hooks") + assert.Len(t, requestLog2.All(), 1) + records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) require.NoError(t, err) require.Len(t, records, 1) @@ -190,6 +193,9 @@ func TestPublishHook(t *testing.T) { requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") assert.Len(t, requestLog.All(), 1) + requestLog2 := xEnv.Observer().FilterMessage("error applying publish event hooks") + assert.Len(t, requestLog2.All(), 1) + msgOne, err := firstSub.NextMsg(5 * time.Second) require.NoError(t, err) require.Equal(t, xEnv.GetPubSubName("employeeUpdatedMyNats.3"), msgOne.Subject) @@ -234,7 +240,10 @@ func TestPublishHook(t *testing.T) { requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") assert.Len(t, requestLog.All(), 1) - + + requestLog2 := xEnv.Observer().FilterMessage("error applying publish event hooks") + assert.Len(t, requestLog2.All(), 1) + require.Len(t, records, 1) }) }) diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 3c2bf97d36..6e1e1c25d7 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -162,12 +162,9 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B } if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { - _, errWrite := io.WriteString(out, `{"success": false}`) - if errWrite != nil { - return errWrite - } - // it will not be returned but only logged to avoid a "unable to fetch from subgraph" error - return nil + // err will not be returned but only logged inside PubSubProvider.Publish to avoid a "unable to fetch from subgraph" error + _, errWrite := io.WriteString(out, `{"success": false}`) + return errWrite } _, errWrite := io.WriteString(out, `{"success": true}`) if errWrite != nil { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index c2467aa753..d89adb110b 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -154,8 +154,9 @@ func (s *NatsPublishDataSource) Load(ctx context.Context, input []byte, out *byt } if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { - _, err = io.WriteString(out, `{"success": false}`) - return err + // err will not be returned but only logged inside PubSubProvider.Publish to avoid a "unable to fetch from subgraph" error + _, errWrite := io.WriteString(out, `{"success": false}`) + return errWrite } _, err := io.WriteString(out, `{"success": true}`) return err diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index b221360383..77d5906951 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -156,8 +156,9 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B } if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { - _, err = io.WriteString(out, `{"success": false}`) - return err + // err will not be returned but only logged inside PubSubProvider.Publish to avoid a "unable to fetch from subgraph" error + _, errWrite := io.WriteString(out, `{"success": false}`) + return errWrite } _, err := io.WriteString(out, `{"success": true}`) return err From 3d829f871e7c0c0830bd46478b2b38f69407e32e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 16:50:56 +0200 Subject: [PATCH 121/173] chore: send updates even when an hook returns an error --- .../pkg/pubsub/datasource/subscription_event_updater.go | 7 ++++--- .../pubsub/datasource/subscription_event_updater_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index b66c586930..22d5b2438c 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -38,6 +38,9 @@ func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { } processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnStreamEvents) + // updates the events even if the hooks fail + // if a hook doesn't want to send the events, it should return no events! + s.updateEvents(processedEvents) if err != nil { // Check if the error is a StreamHookError and should close the subscription // We use type assertion to check for the CloseSubscription method without importing core @@ -59,8 +62,6 @@ func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { } } - s.updateEvents(processedEvents) - return nil } @@ -88,7 +89,7 @@ func applyStreamEventHooks( var err error currentEvents, err = hook(ctx, cfg, currentEvents) if err != nil { - return nil, err + return currentEvents, err } } return currentEvents, nil diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go index 2ca7407e91..96be85e9f8 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater_test.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -563,7 +563,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t // Define hook that returns a StreamHookError with CloseSubscription=true testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { - return nil, mockHookError + return events, mockHookError } updater := &subscriptionEventUpdater{ @@ -575,13 +575,12 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t }, } + mockUpdater.On("Update", []byte("test data")).Return() err := updater.Update(events) // Should return the error when CloseSubscription is true assert.Error(t, err) assert.Equal(t, mockHookError, err) - // Assert that Update was not called on the eventUpdater - mockUpdater.AssertNotCalled(t, "Update") } func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription(t *testing.T) { @@ -604,7 +603,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription // Define hook that returns a StreamHookError with CloseSubscription=false testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { - return nil, mockHookError + return events, mockHookError } updater := &subscriptionEventUpdater{ @@ -616,6 +615,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription }, } + mockUpdater.On("Update", []byte("test data")).Return() err := updater.Update(events) // Should return nil when CloseSubscription is false (error is logged) From 018e53ced04e2f8de4438c8d19c05336c79aa240 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 18:37:31 +0200 Subject: [PATCH 122/173] fix: rootFieldName missing --- router-tests/modules/stream_publish_test.go | 64 ++++++++++++++++++++ router/pkg/pubsub/kafka/engine_datasource.go | 2 +- router/pkg/pubsub/nats/engine_datasource.go | 2 +- router/pkg/pubsub/redis/engine_datasource.go | 2 +- 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 3d46e309e9..118cc1f6f8 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -3,6 +3,7 @@ package module_test import ( "encoding/json" "errors" + "strconv" "testing" "time" @@ -247,4 +248,67 @@ func TestPublishHook(t *testing.T) { require.Len(t, records, 1) }) }) + + t.Run("Test kafka module publish with argument in header", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + if ctx.PublishEventConfiguration().RootFieldName() != "updateEmployeeMyKafka" { + return events, nil + } + + employeeID := ctx.RequestContext().Operation().Variables().GetInt("employeeID") + + newEvents := []datasource.StreamEvent{} + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + if evt.Headers == nil { + evt.Headers = map[string][]byte{} + } + evt.Headers["x-employee-id"] = []byte(strconv.Itoa(employeeID)) + newEvents = append(newEvents, event) + } + return newEvents, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + events.KafkaEnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation UpdateEmployeeKafka($employeeID: Int!) { updateEmployeeMyKafka(employeeID: $employeeID, update: {name: "name test"}) { success } }`, + Variables: json.RawMessage(`{"employeeID": 3}`), + }) + require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": true}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + records, err := events.ReadKafkaMessages(xEnv, time.Second, "employeeUpdated", 1) + require.NoError(t, err) + require.Len(t, records, 1) + header := records[0].Headers[0] + require.Equal(t, "x-employee-id", header.Key) + require.Equal(t, []byte("3"), header.Value) + }) + }) } diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 6e1e1c25d7..28cbcb6d8f 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -78,7 +78,7 @@ func (p *publishData) MarshalJSONTemplate() (string, error) { return "", err } - return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s"}`, p.Topic, p.Event.Data, p.Event.Key, headersBytes, p.Provider), nil + return fmt.Sprintf(`{"topic":"%s", "event": {"data": %s, "key": "%s", "headers": %s}, "providerId":"%s", "rootFieldName":"%s"}`, p.Topic, p.Event.Data, p.Event.Key, headersBytes, p.Provider, p.FieldName), nil } // PublishEventConfiguration is a public type that is used to allow access to custom fields diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index d89adb110b..8c5b34af71 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -71,7 +71,7 @@ func (p *publishData) PublishEventConfiguration() datasource.PublishEventConfigu func (p *publishData) MarshalJSONTemplate() (string, error) { // The content of the data field could be not valid JSON, so we can't use json.Marshal // e.g. {"id":$$0$$,"update":$$1$$} - return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s"}`, p.Subject, p.Event.Data, p.Provider), nil + return fmt.Sprintf(`{"subject":"%s", "event": {"data": %s}, "providerId":"%s", "rootFieldName":"%s"}`, p.Subject, p.Event.Data, p.Provider, p.FieldName), nil } type PublishAndRequestEventConfiguration struct { diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index 77d5906951..b59b9ab0b5 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -63,7 +63,7 @@ func (p *publishData) PublishEventConfiguration() datasource.PublishEventConfigu } func (p *publishData) MarshalJSONTemplate() (string, error) { - return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s"}`, p.Channel, p.Event.Data, p.Provider), nil + return fmt.Sprintf(`{"channel":"%s", "event": {"data": %s}, "providerId":"%s", "rootFieldName":"%s"}`, p.Channel, p.Event.Data, p.Provider, p.FieldName), nil } // PublishEventConfiguration contains configuration for publish events From 3df02fec28f6f0b952e71297c820f9e913149d58 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 8 Aug 2025 18:45:27 +0200 Subject: [PATCH 123/173] chore: fix tests --- router/pkg/pubsub/kafka/engine_datasource_test.go | 12 ++++++++---- router/pkg/pubsub/nats/engine_datasource_test.go | 6 ++++-- router/pkg/pubsub/redis/engine_datasource_test.go | 6 ++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/router/pkg/pubsub/kafka/engine_datasource_test.go b/router/pkg/pubsub/kafka/engine_datasource_test.go index cd7c382a92..846203d6e0 100644 --- a/router/pkg/pubsub/kafka/engine_datasource_test.go +++ b/router/pkg/pubsub/kafka/engine_datasource_test.go @@ -26,8 +26,9 @@ func TestPublishData_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider", Topic: "test-topic", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + FieldName: "test-field", }, - wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}, "key": "", "headers": {}}, "providerId":"test-provider"}`, + wantPattern: `{"topic":"test-topic", "event": {"data": {"message":"hello"}, "key": "", "headers": {}}, "providerId":"test-provider", "rootFieldName":"test-field"}`, }, { name: "with special characters", @@ -35,8 +36,9 @@ func TestPublishData_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + FieldName: "test-field", }, - wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}, "key": "", "headers": {}}, "providerId":"test-provider-id"}`, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}, "key": "", "headers": {}}, "providerId":"test-provider-id", "rootFieldName":"test-field"}`, }, { name: "with key", @@ -44,8 +46,9 @@ func TestPublishData_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Key: []byte("blablabla"), Data: json.RawMessage(`{}`)}, + FieldName: "test-field", }, - wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "blablabla", "headers": {}}, "providerId":"test-provider-id"}`, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "blablabla", "headers": {}}, "providerId":"test-provider-id", "rootFieldName":"test-field"}`, }, { name: "with headers", @@ -53,8 +56,9 @@ func TestPublishData_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider-id", Topic: "topic-with-hyphens", Event: Event{Headers: map[string][]byte{"key": []byte(`blablabla`)}, Data: json.RawMessage(`{}`)}, + FieldName: "test-field", }, - wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "", "headers": {"key":"YmxhYmxhYmxh"}}, "providerId":"test-provider-id"}`, + wantPattern: `{"topic":"topic-with-hyphens", "event": {"data": {}, "key": "", "headers": {"key":"YmxhYmxhYmxh"}}, "providerId":"test-provider-id", "rootFieldName":"test-field"}`, }, } diff --git a/router/pkg/pubsub/nats/engine_datasource_test.go b/router/pkg/pubsub/nats/engine_datasource_test.go index 3e3f30d9c0..8665f42181 100644 --- a/router/pkg/pubsub/nats/engine_datasource_test.go +++ b/router/pkg/pubsub/nats/engine_datasource_test.go @@ -28,8 +28,9 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider", Subject: "test-subject", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + FieldName: "test-field", }, - wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, + wantPattern: `{"subject":"test-subject", "event": {"data": {"message":"hello"}}, "providerId":"test-provider", "rootFieldName":"test-field"}`, }, { name: "with special characters", @@ -37,8 +38,9 @@ func TestPublishAndRequestEventConfiguration_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider-id", Subject: "subject-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + FieldName: "test-field", }, - wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, + wantPattern: `{"subject":"subject-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id", "rootFieldName":"test-field"}`, }, } diff --git a/router/pkg/pubsub/redis/engine_datasource_test.go b/router/pkg/pubsub/redis/engine_datasource_test.go index 558a9a7d65..b322c8a60c 100644 --- a/router/pkg/pubsub/redis/engine_datasource_test.go +++ b/router/pkg/pubsub/redis/engine_datasource_test.go @@ -26,8 +26,9 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider", Channel: "test-channel", Event: Event{Data: json.RawMessage(`{"message":"hello"}`)}, + FieldName: "test-field", }, - wantPattern: `{"channel":"test-channel", "event": {"data": {"message":"hello"}}, "providerId":"test-provider"}`, + wantPattern: `{"channel":"test-channel", "event": {"data": {"message":"hello"}}, "providerId":"test-provider", "rootFieldName":"test-field"}`, }, { name: "with special characters", @@ -35,8 +36,9 @@ func TestPublishEventConfiguration_MarshalJSONTemplate(t *testing.T) { Provider: "test-provider-id", Channel: "channel-with-hyphens", Event: Event{Data: json.RawMessage(`{"message":"special \"quotes\" here"}`)}, + FieldName: "test-field", }, - wantPattern: `{"channel":"channel-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id"}`, + wantPattern: `{"channel":"channel-with-hyphens", "event": {"data": {"message":"special \"quotes\" here"}}, "providerId":"test-provider-id", "rootFieldName":"test-field"}`, }, } From 511a1728c1752a4cb39adc568fd083b78ee407f3 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 19 Aug 2025 15:13:41 +0200 Subject: [PATCH 124/173] chore: remove close option from subscription start hook --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 +- router/core/subscriptions_modules.go | 28 ++------- router/go.mod | 2 +- router/go.sum | 4 +- router/pkg/pubsub/datasource/provider.go | 2 +- .../datasource/subscription_datasource.go | 12 ++-- .../subscription_datasource_test.go | 58 +++++-------------- 8 files changed, 30 insertions(+), 82 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 8c4393e24d..b838c478a1 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 6e37cd1eee..f0e0931089 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb h1:bjlqTclnvdC+C2wjHtgXTjXTqNzZQCk6pJqGUpsHdGI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 8934c463df..55fbfb863e 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -1,8 +1,6 @@ package core import ( - "errors" - "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" @@ -117,7 +115,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext return nil } - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (bool, error) { + return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &pubSubSubscriptionOnStartHookContext{ requestContext: requestContext, @@ -125,16 +123,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } - err := fn(hookCtx) - - // Check if the error is a StreamHookError and should close the connection - var streamHookErr *StreamHookError - close := false - if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseSubscription() - } - - return close, err + return fn(hookCtx) } } @@ -144,22 +133,13 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext return nil } - return func(resolveCtx *resolve.Context, input []byte) (bool, error) { + return func(resolveCtx *resolve.Context, input []byte) (error) { requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, } - err := fn(hookCtx) - - // Check if the error is a StreamHookError and should close the connection - var streamHookErr *StreamHookError - close := false - if errors.As(err, &streamHookErr) { - close = streamHookErr.CloseSubscription() - } - - return close, err + return fn(hookCtx) } } diff --git a/router/go.mod b/router/go.mod index fbc5b5f63f..8a8f5ecdbe 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 9a485f9a1e..79e58e3218 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb h1:bjlqTclnvdC+C2wjHtgXTjXTqNzZQCk6pJqGUpsHdGI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index 0a699e2b14..e34c040b20 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -59,7 +59,7 @@ type StreamEvent interface { GetData() []byte } -type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) (bool, error) +type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) error // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 83ce199ff8..4a9a4b9e43 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -42,19 +42,19 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(updater)) } -func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (close bool, err error) { +func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (err error) { for _, fn := range s.subscriptionOnStartFns { conf, errConf := s.SubscriptionEventConfiguration(input) if errConf != nil { - return true, err + return err } - close, err = fn(ctx, conf) - if err != nil || close { - return + err = fn(ctx, conf) + if err != nil { + return err } } - return + return nil } func (s *PubSubSubscriptionDataSource[C]) SetSubscriptionOnStartFns(fns ...SubscriptionOnStartFn) { diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index bbeee43037..613b2af9ae 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -189,9 +189,8 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_Success(t *testing.T) ctx := &resolve.Context{} - close, err := dataSource.SubscriptionOnStart(ctx, input) + err = dataSource.SubscriptionOnStart(ctx, input) assert.NoError(t, err) - assert.False(t, close) } func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T) { @@ -206,14 +205,14 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T hook1Called := false hook2Called := false - hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { hook1Called = true - return false, nil + return nil } - hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { + hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { hook2Called = true - return false, nil + return nil } dataSource.SetSubscriptionOnStartFns(hook1, hook2) @@ -227,42 +226,12 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T ctx := &resolve.Context{} - close, err := dataSource.SubscriptionOnStart(ctx, input) + err = dataSource.SubscriptionOnStart(ctx, input) assert.NoError(t, err) - assert.False(t, close) assert.True(t, hook1Called) assert.True(t, hook2Called) } -func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsClose(t *testing.T) { - mockAdapter := NewMockProvider(t) - uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { - return nil - } - - dataSource := NewPubSubSubscriptionDataSource[testSubscriptionEventConfiguration](mockAdapter, uniqueRequestIDFn) - - // Add hook that returns close=true - hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { - return true, nil - } - - dataSource.SetSubscriptionOnStartFns(hook) - - testConfig := testSubscriptionEventConfiguration{ - Topic: "test-topic", - Subject: "test-subject", - } - input, err := json.Marshal(testConfig) - assert.NoError(t, err) - - ctx := &resolve.Context{} - - close, err := dataSource.SubscriptionOnStart(ctx, input) - assert.NoError(t, err) - assert.True(t, close) -} - func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *testing.T) { mockAdapter := NewMockProvider(t) uniqueRequestIDFn := func(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) error { @@ -273,8 +242,8 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te expectedError := errors.New("hook error") // Add hook that returns an error - hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { - return false, expectedError + hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + return expectedError } dataSource.SetSubscriptionOnStartFns(hook) @@ -288,10 +257,9 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te ctx := &resolve.Context{} - close, err := dataSource.SubscriptionOnStart(ctx, input) + err = dataSource.SubscriptionOnStart(ctx, input) assert.Error(t, err) assert.Equal(t, expectedError, err) - assert.False(t, close) } func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { @@ -306,11 +274,11 @@ func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { assert.Len(t, dataSource.subscriptionOnStartFns, 0) // Add hooks - hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { - return false, nil + hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + return nil } - hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (bool, error) { - return false, nil + hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + return nil } dataSource.SetSubscriptionOnStartFns(hook1) From e34b6fe80d1acdc66ffe91d22c673d309184c47f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 19 Aug 2025 15:47:27 +0200 Subject: [PATCH 125/173] chore: use updated engine --- router-tests/go.mod | 4 ++-- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index b838c478a1..c5ea243395 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,8 +25,8 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb + github.com/wundergraph/cosmo/router v0.0.0-20250819131341-511a1728c175 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index f0e0931089..0dada5c50f 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb h1:bjlqTclnvdC+C2wjHtgXTjXTqNzZQCk6pJqGUpsHdGI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 8a8f5ecdbe..a3c675c350 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 79e58e3218..a5cedb877f 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb h1:bjlqTclnvdC+C2wjHtgXTjXTqNzZQCk6pJqGUpsHdGI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819130252-6793f09278fb/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 70ef951c83797817aef6da56938edc20c2bdce42 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 19 Aug 2025 15:50:13 +0200 Subject: [PATCH 126/173] chore: use update router --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- router-tests/go.mod | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 3483103043..9a6b0a4ed4 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd + github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 20bd3ed298..c59cea23fe 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd h1:72U7uIdZ5uimYmPh7egT/84rOG8qR6sOdASeI6nuFo4= -github.com/wundergraph/cosmo/router v0.0.0-20250806075617-cd9e4e4994fd/go.mod h1:RBUiLGhXKdJMe9wQA8zDzED71ptNrpL2Wv2obmKfMco= +github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a h1:we5EMiC4y2mKtSVXfkflm+JIJhem9znUr2YqrA/L07o= +github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a/go.mod h1:f9cLvJI8IjDNBCv7iYlboZVsbmw1WwBUvJjdrqGCtno= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313 h1:kKoKImKopqTeLqL3em5VyLhe82tPO8eEYe/ToI8ZHpA= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250806074950-2ed6134af313/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/go.mod b/router-tests/go.mod index c5ea243395..8ced9870e4 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250819131341-511a1728c175 + github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 From fb6c1a46d9e0987bda4bae0a531cc277c9dd7d7e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 19 Aug 2025 18:31:32 +0200 Subject: [PATCH 127/173] chore: fix test for new behaviour when an error is returned --- router-tests/modules/start_subscription_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 4a91a80132..2be3fb72c4 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -417,7 +417,8 @@ func TestStartSubscriptionHook(t *testing.T) { clientRunCh <- client.Run() }() - xEnv.WaitForSubscriptionCount(1, time.Second*10) + // Wait for the subscription to be closed + xEnv.WaitForSubscriptionCount(0, time.Second*10) testenv.AwaitChannelWithT(t, time.Second*10, subscriptionOneArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { var graphqlErrs graphql.Errors From 103443b5b16f4b2d692bab422b54fe1445425373 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 20 Aug 2025 16:38:28 +0200 Subject: [PATCH 128/173] chore: remove closeSubscription field from StreamHookError and update related tests --- router-tests/modules/start_subscription_test.go | 9 ++++++--- router/core/subscriptions_modules.go | 9 +-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 2be3fb72c4..dd07d07a85 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -91,6 +91,9 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { + if ctx.SubscriptionEventConfiguration().RootFieldName() != "employeeUpdatedMyKafka" { + return nil + } ctx.WriteEvent(&kafka.Event{ Key: []byte("1"), Data: []byte(`{"id": 1, "__typename": "Employee"}`), @@ -178,7 +181,7 @@ func TestStartSubscriptionHook(t *testing.T) { "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { callbackCalled <- true - return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "", true) + return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "") }, }, }, @@ -363,7 +366,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { - return core.NewStreamHookError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected), false) + return core.NewStreamHookError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected)) }, }, }, @@ -591,7 +594,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { - return core.NewStreamHookError(errors.New("subscription closed"), "subscription closed", http.StatusOK, "NotFound", true) + return core.NewStreamHookError(errors.New("subscription closed"), "subscription closed", http.StatusOK, "NotFound") }, CallbackOnOriginResponse: func(response *http.Response, ctx core.RequestContext) *http.Response { originResponseCalled <- response diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 55fbfb863e..35fb8e088b 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -12,7 +12,6 @@ type StreamHookError struct { message string statusCode int code string - closeSubscription bool } func (e *StreamHookError) Error() string { @@ -34,17 +33,12 @@ func (e *StreamHookError) Code() string { return e.code } -func (e *StreamHookError) CloseSubscription() bool { - return e.closeSubscription -} - -func NewStreamHookError(err error, message string, statusCode int, code string, closeSubscription bool) *StreamHookError { +func NewStreamHookError(err error, message string, statusCode int, code string) *StreamHookError { return &StreamHookError{ err: err, message: message, statusCode: statusCode, code: code, - closeSubscription: closeSubscription, } } @@ -104,7 +98,6 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() type SubscriptionOnStartHandler interface { // SubscriptionOnStart is called once at subscription start - // If the error is a StreamHookError and CloseSubscription is true, the subscription is closed. // The error is propagated to the client. SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } From 384d0279ba5eebf711be4d768671f3bfbbf4dc5e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 20 Aug 2025 18:11:39 +0200 Subject: [PATCH 129/173] chore: update to new engine --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router-tests/modules/start_subscription_test.go | 2 +- router/core/subscriptions_modules.go | 16 ++++++++++------ router/go.mod | 2 +- router/go.sum | 4 ++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 8ced9870e4..d8a2feff0a 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 0dada5c50f..79f1c4ecce 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a h1:izz76F24vKnoSg5ctxud8qDpd6AZUCxp0wkfq03YqBo= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index dd07d07a85..bdfbfa9105 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -585,7 +585,7 @@ func TestStartSubscriptionHook(t *testing.T) { }) }) - t.Run("Test StartSubscription hook is called, return StreamHookError with CloseConnection true, response on OnOriginResponse should still be set", func(t *testing.T) { + t.Run("Test StartSubscription hook is called, return StreamHookError, response on OnOriginResponse should still be set", func(t *testing.T) { t.Parallel() originResponseCalled := make(chan *http.Response, 1) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 35fb8e088b..ec87b66b6d 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -55,7 +55,7 @@ type SubscriptionOnStartHookContext interface { type pubSubSubscriptionOnStartHookContext struct { requestContext RequestContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration - writeEventHook func(data []byte) bool + writeEventHook func(data []byte) } func (c *pubSubSubscriptionOnStartHookContext) RequestContext() RequestContext { @@ -67,7 +67,9 @@ func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() } func (c *pubSubSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { - return c.writeEventHook(event.GetData()) + c.writeEventHook(event.GetData()) + + return true } // EngineEvent is the event used to write to the engine subscription @@ -81,7 +83,7 @@ func (e *EngineEvent) GetData() []byte { type engineSubscriptionOnStartHookContext struct { requestContext RequestContext - writeEventHook func(data []byte) bool + writeEventHook func(data []byte) } func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { @@ -89,7 +91,9 @@ func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { } func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { - return c.writeEventHook(event.GetData()) + c.writeEventHook(event.GetData()) + + return true } func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { @@ -113,7 +117,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext hookCtx := &pubSubSubscriptionOnStartHookContext{ requestContext: requestContext, subscriptionEventConfiguration: subConf, - writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, + writeEventHook: resolveCtx.EmitSubscriptionUpdate, } return fn(hookCtx) @@ -130,7 +134,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext requestContext := getRequestContext(resolveCtx.Context()) hookCtx := &engineSubscriptionOnStartHookContext{ requestContext: requestContext, - writeEventHook: resolveCtx.TryEmitSubscriptionUpdate, + writeEventHook: resolveCtx.EmitSubscriptionUpdate, } return fn(hookCtx) diff --git a/router/go.mod b/router/go.mod index a3c675c350..ae28a77130 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index a5cedb877f..ab03d317c7 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a h1:izz76F24vKnoSg5ctxud8qDpd6AZUCxp0wkfq03YqBo= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 8177f80ddd5e4c57087272d006828f2f68ac7dc0 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 21 Aug 2025 10:15:23 +0200 Subject: [PATCH 130/173] chore: update engine --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index d8a2feff0a..df3e7736da 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 79f1c4ecce..3a0bc40128 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a h1:izz76F24vKnoSg5ctxud8qDpd6AZUCxp0wkfq03YqBo= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 h1:JEx2DeqFMX9mgbpdkRJt83u4OvhoHPizWI8SLxLpNjg= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index ae28a77130..2889008283 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index ab03d317c7..c4c9a26dd8 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a h1:izz76F24vKnoSg5ctxud8qDpd6AZUCxp0wkfq03YqBo= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250820160909-447c103daf3a/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 h1:JEx2DeqFMX9mgbpdkRJt83u4OvhoHPizWI8SLxLpNjg= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 0614486277cb285e731532b9f24329f3137316ff Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 21 Aug 2025 10:18:05 +0200 Subject: [PATCH 131/173] chore: update graphql-go-tools dependency to latest version --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index df3e7736da..46d22d127d 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 3a0bc40128..4b96eb3867 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 h1:JEx2DeqFMX9mgbpdkRJt83u4OvhoHPizWI8SLxLpNjg= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b h1:6NqzGDxAeAJr/7DMkXOs+YUPQuaSh5+HBL2KL13fiJo= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 2889008283..09b4661749 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index c4c9a26dd8..147d450df7 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45 h1:JEx2DeqFMX9mgbpdkRJt83u4OvhoHPizWI8SLxLpNjg= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821080942-7bb99a5cce45/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b h1:6NqzGDxAeAJr/7DMkXOs+YUPQuaSh5+HBL2KL13fiJo= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From f3c67c7dcfb3ad690c2064afb7177e428cc0c64a Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 21 Aug 2025 13:04:40 +0200 Subject: [PATCH 132/173] chore: update graphql-go-tools dependency to new release version --- router-tests/go.mod | 2 +- router-tests/go.sum | 2 ++ router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 46d22d127d..2bb5eecccb 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 4b96eb3867..e294ffcffd 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -327,6 +327,8 @@ github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoT github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b h1:6NqzGDxAeAJr/7DMkXOs+YUPQuaSh5+HBL2KL13fiJo= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 09b4661749..dead0af4d9 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 147d450df7..9cfc7ed47a 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b h1:6NqzGDxAeAJr/7DMkXOs+YUPQuaSh5+HBL2KL13fiJo= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 4a8bcbd5fe9d751d00ab1cccbee3614a717cc3f0 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 21 Aug 2025 16:44:06 +0200 Subject: [PATCH 133/173] chore: update router and graphql-go-tools dependencies to latest versions --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- router-tests/go.mod | 2 +- router-tests/go.sum | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 9a6b0a4ed4..af602af573 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a + github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index c59cea23fe..0839713615 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a h1:we5EMiC4y2mKtSVXfkflm+JIJhem9znUr2YqrA/L07o= -github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a/go.mod h1:f9cLvJI8IjDNBCv7iYlboZVsbmw1WwBUvJjdrqGCtno= +github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 h1:DjsaoBtR/FUhYAIGz9CNIuwZIovXwJn+UEXKg+0BgrU= +github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3/go.mod h1:GluINInx09636UxKONkkfk3/Y5zU4ueX9ru4z9Pt100= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c h1:mjTOMnaUw86cwjF0o1882bI327aNT5wr80h6oETsW+4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250819134450-fdf829add23c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/go.mod b/router-tests/go.mod index 2bb5eecccb..76c7ccdbdc 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250819134727-e34b6fe80d1a + github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index e294ffcffd..f04770acc6 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,6 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b h1:6NqzGDxAeAJr/7DMkXOs+YUPQuaSh5+HBL2KL13fiJo= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821081603-f8c9d0684b0b/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= From aa67e6d482f6f2064db44677ce5d4f16190b846d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Fri, 22 Aug 2025 18:24:37 +0200 Subject: [PATCH 134/173] chore: make the SubscriptionOnStartHookContext more restricted --- .../modules/start_subscription_test.go | 2 +- .../cmd/router/start-subscription/module.go | 46 ++++++++++++++ router/core/subscriptions_modules.go | 63 ++++++++++++++----- router/pkg/pubsub/datasource/provider.go | 2 +- .../datasource/subscription_datasource.go | 2 +- .../subscription_datasource_test.go | 25 +++++--- 6 files changed, 113 insertions(+), 27 deletions(-) create mode 100644 router/cmd/router/start-subscription/module.go diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index bdfbfa9105..ad286d54ef 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -262,7 +262,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHookContext) error { - employeeId := ctx.RequestContext().Operation().Variables().GetInt64("employeeID") + employeeId := ctx.Operation().Variables().GetInt64("employeeID") if employeeId != 1 { return nil } diff --git a/router/cmd/router/start-subscription/module.go b/router/cmd/router/start-subscription/module.go new file mode 100644 index 0000000000..dd4bc7d978 --- /dev/null +++ b/router/cmd/router/start-subscription/module.go @@ -0,0 +1,46 @@ +package startsubscription + +import ( + "go.uber.org/zap" + + "github.com/wundergraph/cosmo/router/core" +) + +func init() { + // Register your module here + core.RegisterModule(&SubscriptionModule{}) +} + +const ( + ModuleID = "com.example.start-subscription" +) + +type SubscriptionModule struct { + Logger *zap.Logger +} + +func (m *SubscriptionModule) Provision(ctx *core.ModuleContext) error { + // Assign the logger to the module for non-request related logging + m.Logger = ctx.Logger + + return nil +} + +func (m *SubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { + m.Logger.Info("SubscriptionOnStart") + return core.NewStreamHookError(nil, "test", 200, "test") +} + +func (m *SubscriptionModule) Module() core.ModuleInfo { + return core.ModuleInfo{ + // This is the ID of your module, it must be unique + ID: ModuleID, + New: func() core.Module { + return &SubscriptionModule{} + }, + } +} + +var _ interface { + core.SubscriptionOnStartHandler +} = (*SubscriptionModule)(nil) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index ec87b66b6d..42b3e56705 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -1,9 +1,12 @@ package core import ( + "net/http" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" + "go.uber.org/zap" ) // StreamHookError is used to customize the error messages and the behavior @@ -43,8 +46,12 @@ func NewStreamHookError(err error, message string, statusCode int, code string) } type SubscriptionOnStartHookContext interface { - // the request context - RequestContext() RequestContext + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext // the subscription event configuration (will return nil for engine subscription) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration // write an event to the stream of the current subscription @@ -53,13 +60,23 @@ type SubscriptionOnStartHookContext interface { } type pubSubSubscriptionOnStartHookContext struct { - requestContext RequestContext + request *http.Request + logger *zap.Logger + operation OperationContext subscriptionEventConfiguration datasource.SubscriptionEventConfiguration writeEventHook func(data []byte) } -func (c *pubSubSubscriptionOnStartHookContext) RequestContext() RequestContext { - return c.requestContext +func (c *pubSubSubscriptionOnStartHookContext) Request() *http.Request { + return c.request +} + +func (c *pubSubSubscriptionOnStartHookContext) Logger() *zap.Logger { + return c.logger +} + +func (c *pubSubSubscriptionOnStartHookContext) Operation() OperationContext { + return c.operation } func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { @@ -82,12 +99,22 @@ func (e *EngineEvent) GetData() []byte { } type engineSubscriptionOnStartHookContext struct { - requestContext RequestContext + request *http.Request + logger *zap.Logger + operation OperationContext writeEventHook func(data []byte) } -func (c *engineSubscriptionOnStartHookContext) RequestContext() RequestContext { - return c.requestContext +func (c *engineSubscriptionOnStartHookContext) Request() *http.Request { + return c.request +} + +func (c *engineSubscriptionOnStartHookContext) Logger() *zap.Logger { + return c.logger +} + +func (c *engineSubscriptionOnStartHookContext) Operation() OperationContext { + return c.operation } func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { @@ -112,12 +139,14 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext return nil } - return func(resolveCtx *resolve.Context, subConf datasource.SubscriptionEventConfiguration) (error) { - requestContext := getRequestContext(resolveCtx.Context()) + return func(resolveCtx resolve.StartupHookContext, subConf datasource.SubscriptionEventConfiguration) error { + requestContext := getRequestContext(resolveCtx.Context) hookCtx := &pubSubSubscriptionOnStartHookContext{ - requestContext: requestContext, + request: requestContext.Request(), + logger: requestContext.Logger(), + operation: requestContext.Operation(), subscriptionEventConfiguration: subConf, - writeEventHook: resolveCtx.EmitSubscriptionUpdate, + writeEventHook: resolveCtx.Updater, } return fn(hookCtx) @@ -130,11 +159,13 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext return nil } - return func(resolveCtx *resolve.Context, input []byte) (error) { - requestContext := getRequestContext(resolveCtx.Context()) + return func(resolveCtx resolve.StartupHookContext, input []byte) error { + requestContext := getRequestContext(resolveCtx.Context) hookCtx := &engineSubscriptionOnStartHookContext{ - requestContext: requestContext, - writeEventHook: resolveCtx.EmitSubscriptionUpdate, + request: requestContext.Request(), + logger: requestContext.Logger(), + operation: requestContext.Operation(), + writeEventHook: resolveCtx.Updater, } return fn(hookCtx) diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index e34c040b20..ccf450b936 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -59,7 +59,7 @@ type StreamEvent interface { GetData() []byte } -type SubscriptionOnStartFn func(ctx *resolve.Context, subConf SubscriptionEventConfiguration) error +type SubscriptionOnStartFn func(ctx resolve.StartupHookContext, subConf SubscriptionEventConfiguration) error // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement type SubscriptionEventConfiguration interface { diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 4a9a4b9e43..e5c9c26ab6 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -42,7 +42,7 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(updater)) } -func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx *resolve.Context, input []byte) (err error) { +func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx resolve.StartupHookContext, input []byte) (err error) { for _, fn := range s.subscriptionOnStartFns { conf, errConf := s.SubscriptionEventConfiguration(input) if errConf != nil { diff --git a/router/pkg/pubsub/datasource/subscription_datasource_test.go b/router/pkg/pubsub/datasource/subscription_datasource_test.go index 613b2af9ae..a9170d7edd 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource_test.go +++ b/router/pkg/pubsub/datasource/subscription_datasource_test.go @@ -187,7 +187,10 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_Success(t *testing.T) input, err := json.Marshal(testConfig) assert.NoError(t, err) - ctx := &resolve.Context{} + ctx := resolve.StartupHookContext{ + Context: context.Background(), + Updater: func(data []byte) {}, + } err = dataSource.SubscriptionOnStart(ctx, input) assert.NoError(t, err) @@ -205,12 +208,12 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T hook1Called := false hook2Called := false - hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + hook1 := func(ctx resolve.StartupHookContext, config SubscriptionEventConfiguration) error { hook1Called = true return nil } - hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + hook2 := func(ctx resolve.StartupHookContext, config SubscriptionEventConfiguration) error { hook2Called = true return nil } @@ -224,7 +227,10 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_WithHooks(t *testing.T input, err := json.Marshal(testConfig) assert.NoError(t, err) - ctx := &resolve.Context{} + ctx := resolve.StartupHookContext{ + Context: context.Background(), + Updater: func(data []byte) {}, + } err = dataSource.SubscriptionOnStart(ctx, input) assert.NoError(t, err) @@ -242,7 +248,7 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te expectedError := errors.New("hook error") // Add hook that returns an error - hook := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + hook := func(ctx resolve.StartupHookContext, config SubscriptionEventConfiguration) error { return expectedError } @@ -255,7 +261,10 @@ func TestPubSubSubscriptionDataSource_SubscriptionOnStart_HookReturnsError(t *te input, err := json.Marshal(testConfig) assert.NoError(t, err) - ctx := &resolve.Context{} + ctx := resolve.StartupHookContext{ + Context: context.Background(), + Updater: func(data []byte) {}, + } err = dataSource.SubscriptionOnStart(ctx, input) assert.Error(t, err) @@ -274,10 +283,10 @@ func TestPubSubSubscriptionDataSource_SetSubscriptionOnStartFns(t *testing.T) { assert.Len(t, dataSource.subscriptionOnStartFns, 0) // Add hooks - hook1 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + hook1 := func(ctx resolve.StartupHookContext, config SubscriptionEventConfiguration) error { return nil } - hook2 := func(ctx *resolve.Context, config SubscriptionEventConfiguration) (error) { + hook2 := func(ctx resolve.StartupHookContext, config SubscriptionEventConfiguration) error { return nil } From 5caae33a64f172a2ae28aa43ee4ec4357562ffb6 Mon Sep 17 00:00:00 2001 From: StarpTech Date: Sat, 23 Aug 2025 12:24:23 +0200 Subject: [PATCH 135/173] chore: fix compile issues --- router/cmd/router/main.go | 1 + router/cmd/router/start-subscription/module.go | 3 ++- router/go.mod | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/router/cmd/router/main.go b/router/cmd/router/main.go index 649908e827..af0c142ab6 100644 --- a/router/cmd/router/main.go +++ b/router/cmd/router/main.go @@ -4,6 +4,7 @@ import ( "os" routercmd "github.com/wundergraph/cosmo/router/cmd" + _ "github.com/wundergraph/cosmo/router/cmd/router/start-subscription" ) func main() { diff --git a/router/cmd/router/start-subscription/module.go b/router/cmd/router/start-subscription/module.go index dd4bc7d978..a48c7f0db3 100644 --- a/router/cmd/router/start-subscription/module.go +++ b/router/cmd/router/start-subscription/module.go @@ -28,7 +28,8 @@ func (m *SubscriptionModule) Provision(ctx *core.ModuleContext) error { func (m *SubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { m.Logger.Info("SubscriptionOnStart") - return core.NewStreamHookError(nil, "test", 200, "test") + + return nil } func (m *SubscriptionModule) Module() core.ModuleInfo { diff --git a/router/go.mod b/router/go.mod index dead0af4d9..1e519c85fd 100644 --- a/router/go.mod +++ b/router/go.mod @@ -162,4 +162,4 @@ require ( // Remember you can use Go workspaces to avoid using replace directives in multiple go.mod files // Use what is best for your personal workflow. See CONTRIBUTING.md for more information -// replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 + replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 From 075681946dc6644c808592d28283a9952bb9c035 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 08:42:40 +0200 Subject: [PATCH 136/173] chore: remove test module --- router/cmd/router/main.go | 1 - .../cmd/router/start-subscription/module.go | 47 ------------------- 2 files changed, 48 deletions(-) delete mode 100644 router/cmd/router/start-subscription/module.go diff --git a/router/cmd/router/main.go b/router/cmd/router/main.go index af0c142ab6..649908e827 100644 --- a/router/cmd/router/main.go +++ b/router/cmd/router/main.go @@ -4,7 +4,6 @@ import ( "os" routercmd "github.com/wundergraph/cosmo/router/cmd" - _ "github.com/wundergraph/cosmo/router/cmd/router/start-subscription" ) func main() { diff --git a/router/cmd/router/start-subscription/module.go b/router/cmd/router/start-subscription/module.go deleted file mode 100644 index a48c7f0db3..0000000000 --- a/router/cmd/router/start-subscription/module.go +++ /dev/null @@ -1,47 +0,0 @@ -package startsubscription - -import ( - "go.uber.org/zap" - - "github.com/wundergraph/cosmo/router/core" -) - -func init() { - // Register your module here - core.RegisterModule(&SubscriptionModule{}) -} - -const ( - ModuleID = "com.example.start-subscription" -) - -type SubscriptionModule struct { - Logger *zap.Logger -} - -func (m *SubscriptionModule) Provision(ctx *core.ModuleContext) error { - // Assign the logger to the module for non-request related logging - m.Logger = ctx.Logger - - return nil -} - -func (m *SubscriptionModule) SubscriptionOnStart(ctx core.SubscriptionOnStartHookContext) error { - m.Logger.Info("SubscriptionOnStart") - - return nil -} - -func (m *SubscriptionModule) Module() core.ModuleInfo { - return core.ModuleInfo{ - // This is the ID of your module, it must be unique - ID: ModuleID, - New: func() core.Module { - return &SubscriptionModule{} - }, - } -} - -var _ interface { - core.SubscriptionOnStartHandler -} = (*SubscriptionModule)(nil) From 3ac47c4e20390dd71949b8741767ca22597d97a6 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 09:29:43 +0200 Subject: [PATCH 137/173] chore: restore Authentication on SubscriptionOnStart context --- router/core/subscriptions_modules.go | 21 ++++++++++++++++++--- router/go.mod | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 42b3e56705..505bbfc44f 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -3,6 +3,7 @@ package core import ( "net/http" + "github.com/wundergraph/cosmo/router/pkg/authentication" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/datasource/graphql_datasource" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" @@ -52,10 +53,12 @@ type SubscriptionOnStartHookContext interface { Logger() *zap.Logger // Operation is the GraphQL operation Operation() OperationContext - // the subscription event configuration (will return nil for engine subscription) + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // SubscriptionEventConfiguration is the subscription event configuration (will return nil for engine subscription) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration - // write an event to the stream of the current subscription - // returns true if the event was written to the stream, false if the event was dropped + // WriteEvent writes an event to the stream of the current subscription + // It returns true if the event was written to the stream, false if the event was dropped WriteEvent(event datasource.StreamEvent) bool } @@ -63,6 +66,7 @@ type pubSubSubscriptionOnStartHookContext struct { request *http.Request logger *zap.Logger operation OperationContext + authentication authentication.Authentication subscriptionEventConfiguration datasource.SubscriptionEventConfiguration writeEventHook func(data []byte) } @@ -79,6 +83,10 @@ func (c *pubSubSubscriptionOnStartHookContext) Operation() OperationContext { return c.operation } +func (c *pubSubSubscriptionOnStartHookContext) Authentication() authentication.Authentication { + return c.authentication +} + func (c *pubSubSubscriptionOnStartHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { return c.subscriptionEventConfiguration } @@ -102,6 +110,7 @@ type engineSubscriptionOnStartHookContext struct { request *http.Request logger *zap.Logger operation OperationContext + authentication authentication.Authentication writeEventHook func(data []byte) } @@ -117,6 +126,10 @@ func (c *engineSubscriptionOnStartHookContext) Operation() OperationContext { return c.operation } +func (c *engineSubscriptionOnStartHookContext) Authentication() authentication.Authentication { + return c.authentication +} + func (c *engineSubscriptionOnStartHookContext) WriteEvent(event datasource.StreamEvent) bool { c.writeEventHook(event.GetData()) @@ -145,6 +158,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext request: requestContext.Request(), logger: requestContext.Logger(), operation: requestContext.Operation(), + authentication: requestContext.Authentication(), subscriptionEventConfiguration: subConf, writeEventHook: resolveCtx.Updater, } @@ -165,6 +179,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext request: requestContext.Request(), logger: requestContext.Logger(), operation: requestContext.Operation(), + authentication: requestContext.Authentication(), writeEventHook: resolveCtx.Updater, } diff --git a/router/go.mod b/router/go.mod index 1e519c85fd..dead0af4d9 100644 --- a/router/go.mod +++ b/router/go.mod @@ -162,4 +162,4 @@ require ( // Remember you can use Go workspaces to avoid using replace directives in multiple go.mod files // Use what is best for your personal workflow. See CONTRIBUTING.md for more information - replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 +// replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 From 665269e1abf3eee42d24256f8b17507d9c107144 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 09:31:20 +0200 Subject: [PATCH 138/173] chore: update graphql-go-tools dependency to new release version --- router/go.mod | 2 +- router/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/go.mod b/router/go.mod index dead0af4d9..a3fdb7da47 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 9cfc7ed47a..bf9e5c6cfd 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 h1:diyWyKSWr0ilNgwl6VwuqB5MU4i6rqI0aQPUlDjyKfY= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 806a9cfa886696694942c863b9398b07d2b9353b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 09:35:38 +0200 Subject: [PATCH 139/173] chore: update graphql-go-tools dependency to latest release version --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 76c7ccdbdc..14da2e620f 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index f04770acc6..a223d9d67f 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 h1:diyWyKSWr0ilNgwl6VwuqB5MU4i6rqI0aQPUlDjyKfY= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 31d35e99379aa129dbf9985b2eb95bb7d34fa64e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 11:59:00 +0200 Subject: [PATCH 140/173] chore: update graphql-go-tools dependency to new release version --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 14da2e620f..ed839d6f85 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index a223d9d67f..b6eb94d13b 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 h1:diyWyKSWr0ilNgwl6VwuqB5MU4i6rqI0aQPUlDjyKfY= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index a3fdb7da47..5c5b68e90e 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index bf9e5c6cfd..99adb47559 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1 h1:diyWyKSWr0ilNgwl6VwuqB5MU4i6rqI0aQPUlDjyKfY= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825072956-e0af0a0daee1/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From ba233816f022e99c2baa38740743429055108f75 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 25 Aug 2025 12:00:24 +0200 Subject: [PATCH 141/173] chore: update router and graphql-go-tools dependencies to latest versions --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index af602af573..c2918aed63 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 + github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 0839713615..05940429e9 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 h1:DjsaoBtR/FUhYAIGz9CNIuwZIovXwJn+UEXKg+0BgrU= -github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3/go.mod h1:GluINInx09636UxKONkkfk3/Y5zU4ueX9ru4z9Pt100= +github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a h1:pxpUrzOSSfqsDL+NqoCtxonHvroSCJMtZYTcNBUo6mA= +github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a/go.mod h1:+RsJfcGuc+9hczpy1F1XqPQ23l4ifFwTrKk1v6lSmvY= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73 h1:G9TNXmnAo5xE7P6E+vZlqMJiq4BuaLArdKicg+WvFws= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250821104204-baec1daedd73/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 4a4fdd0a3c379e2265d8bb97c50a46ec3e7f5d96 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 15:20:58 +0200 Subject: [PATCH 142/173] chore: upgrade engine --- router/go.mod | 2 +- router/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/go.mod b/router/go.mod index 5c5b68e90e..ef217b466f 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 99adb47559..95e34346bc 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 14b50070d2a1cb9d727a33560f753cdf7b470784 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 15:22:42 +0200 Subject: [PATCH 143/173] chore: upgrade deps --- demo/go.mod | 4 ++-- demo/go.sum | 3 +++ router-tests/go.mod | 4 ++-- router-tests/go.sum | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index c2918aed63..9ba337b398 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a + github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 05940429e9..7ce5dfbcdd 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -351,10 +351,13 @@ github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-1 github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a h1:pxpUrzOSSfqsDL+NqoCtxonHvroSCJMtZYTcNBUo6mA= github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a/go.mod h1:+RsJfcGuc+9hczpy1F1XqPQ23l4ifFwTrKk1v6lSmvY= +github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 h1:6YOv22fKi3mPk87cd4Np/1JLSxOpxe7iXgS/k+76jiU= +github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37/go.mod h1:IuFXTDd6INokZgeD1WhqAgsK49la9kfkpC9kBRlypJM= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router-tests/go.mod b/router-tests/go.mod index ed839d6f85..721c4c3bba 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,8 +25,8 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250821110440-f3c67c7dcfb3 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 + github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index b6eb94d13b..c177ff1b7c 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From d046fcd5c8dd16bea2973cc74bfe4023fcd5b01f Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 15:28:50 +0200 Subject: [PATCH 144/173] chore: tidy modules --- demo/go.sum | 5 +---- router-tests/go.mod | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/demo/go.sum b/demo/go.sum index 7ce5dfbcdd..6270f5661c 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,14 +349,11 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a h1:pxpUrzOSSfqsDL+NqoCtxonHvroSCJMtZYTcNBUo6mA= -github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a/go.mod h1:+RsJfcGuc+9hczpy1F1XqPQ23l4ifFwTrKk1v6lSmvY= github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 h1:6YOv22fKi3mPk87cd4Np/1JLSxOpxe7iXgS/k+76jiU= github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37/go.mod h1:IuFXTDd6INokZgeD1WhqAgsK49la9kfkpC9kBRlypJM= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 h1:v/YrpaN3MdJmS/E9jH4P62oL12shYPqgjoG0/RkLbXU= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= diff --git a/router-tests/go.mod b/router-tests/go.mod index 721c4c3bba..6b1ae9b0bb 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a + github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 From bd4263aa71190a9ae85351fd769d11cd14e82791 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 16:08:28 +0200 Subject: [PATCH 145/173] chore: updage engine --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 6b1ae9b0bb..c7bca70e56 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index c177ff1b7c..f2ebf2765a 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index ef217b466f..042b78ae76 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 95e34346bc..9139e5a391 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From 02e79be45c68abee0c44d79b7aa2057758214c73 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 16:10:22 +0200 Subject: [PATCH 146/173] chore: update router and graphql-go-tools dependencies to latest versions --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index 9ba337b398..f4451d9c15 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 + github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index 6270f5661c..b83560daa9 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 h1:6YOv22fKi3mPk87cd4Np/1JLSxOpxe7iXgS/k+76jiU= -github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37/go.mod h1:IuFXTDd6INokZgeD1WhqAgsK49la9kfkpC9kBRlypJM= +github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 h1:/b49oS8INO2lhtWpgTX/7hGllWcDNS4gewjKb9UryhM= +github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119/go.mod h1:VEC0JaiosXitFdELqol8Y8ENo1N1ksHMmRzlb87EooQ= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c h1:2jZX/dpnP5S3i0rHB1o0Vmt0QDEID2W3B5QWuhRvago= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903131833-6234791bbf3c/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 7f930035c108023bf58f72f496aeb0b6900fefae Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 3 Sep 2025 16:13:38 +0200 Subject: [PATCH 147/173] chore: go mod tidy --- router-tests/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index c7bca70e56..e4949cd47e 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250903132058-4a4fdd0a3c37 + github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 From 90dba997c7117241151cae9b82ce30651e5ae3be Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 9 Sep 2025 18:23:18 +0200 Subject: [PATCH 148/173] chore: update graphql-go-tools dependency to latest version --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index e4949cd47e..40098128e1 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index f2ebf2765a..4a1ba9c1f1 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e h1:IJ2VFXTW2m6cmethfzwwEfqwf1voHs0QEzdphwY7ViM= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 042b78ae76..7ac0e475b7 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 9139e5a391..927d7c3d1c 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e h1:IJ2VFXTW2m6cmethfzwwEfqwf1voHs0QEzdphwY7ViM= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From e0583e81564bedd7f48c018cde633a1138997bbc Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 9 Sep 2025 18:25:13 +0200 Subject: [PATCH 149/173] chore: update router and graphql-go-tools dependencies to latest versions --- demo/go.mod | 4 ++-- demo/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/demo/go.mod b/demo/go.mod index f4451d9c15..0ffb6aab41 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/cors v1.11.0 github.com/vektah/gqlparser/v2 v2.5.30 github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d - github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 + github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711 github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.28.0 @@ -135,7 +135,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 // indirect github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 // indirect github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e // indirect - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d // indirect + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/demo/go.sum b/demo/go.sum index b83560daa9..98b7936559 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -349,12 +349,12 @@ github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d h github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d/go.mod h1:9I3gPMAlAY+m1/cFL20iN7XHTyuZd3VT5ijccdU/FsI= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e h1:VdJNlsiyWYxJzAD3jEe+DAQdzxkf9btD8qQNYNU+xQU= github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e/go.mod h1:WZ0yBeaDSGHqDMcQrP1JRYgCj9atF7ORXF8srnd2Sro= -github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 h1:/b49oS8INO2lhtWpgTX/7hGllWcDNS4gewjKb9UryhM= -github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119/go.mod h1:VEC0JaiosXitFdELqol8Y8ENo1N1ksHMmRzlb87EooQ= +github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711 h1:qVumNAyqgOowvF5B9pJzmugMDnjGZPJeeeRd4iklTls= +github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711/go.mod h1:8Gd9REvK3o5seHEsU09UDzjknVyterXXrRYrx/41BMY= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f h1:AB3PcgliYMtTInM1Tz1uBbL9eTnGagdiFdyQilJiAIA= github.com/wundergraph/cosmo/router-tests v0.0.0-20250718080012-21acbd83c77f/go.mod h1:ESnTrSqgo+ZcJhB7dFEI3A7T/KaTuA61HLerhcQOXy4= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d h1:BwMY8o9tWmr/nMTBH/zlT/vO/hu512LxP7IOhXeIqs8= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250903140441-8df1b67bce1d/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e h1:IJ2VFXTW2m6cmethfzwwEfqwf1voHs0QEzdphwY7ViM= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From 564837a4b2de37a662b98e666ef802b84edae707 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Mon, 15 Sep 2025 09:01:35 +0200 Subject: [PATCH 150/173] chore: go mod tidy --- router-tests/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 40098128e1..337cfcc049 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,7 +25,7 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250903140828-bd4263aa7119 + github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711 github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 From 2af75cbecf018702ff8a728ce9e91e133b46831b Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 16 Sep 2025 12:34:16 +0200 Subject: [PATCH 151/173] chore: update ADR to reflect implementation of SubscriptionOnStart --- adr/cosmo-streams-v1.md | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 307913aa60..436dafe45b 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -68,14 +68,19 @@ type PublishEventConfiguration interface { } type SubscriptionOnStartHookContext interface { - // the request context - RequestContext() RequestContext - // the stream context - StreamContext() StreamContext - // the subscription event configuration - SubscriptionEventConfiguration() SubscriptionEventConfiguration - // write an event to the stream of the current subscription - WriteEvent(event core.StreamEvent) + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // SubscriptionEventConfiguration is the subscription event configuration (will return nil for engine subscription) + SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration + // WriteEvent writes an event to the stream of the current subscription + // It returns true if the event was written to the stream, false if the event was dropped + WriteEvent(event datasource.StreamEvent) bool } type SubscriptionOnStartHandler interface { @@ -142,7 +147,7 @@ The developer will start by adding a subscription to the cosmo streams graphql s ```graphql type Subscription { - employeeUpdates(): Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") + employeeUpdates: Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") } type Employee @key(fields: "id", resolvable: false) { @@ -286,7 +291,7 @@ The developer will start by adding a subscription to the cosmo streams graphql s ```graphql type Subscription { - employeeUpdates(): Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") + employeeUpdates: Employee! @edfs__natsSubscribe(subjects: ["employeeUpdates"], providerId: "my-nats") } type Employee @key(fields: "id", resolvable: false) { @@ -318,12 +323,12 @@ type MyModule struct {} func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { // check if the provider is nats - if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return nil } // check if the provider id is the one expected by the module - if ctx.StreamContext().ProviderID() != "my-nats" { + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-nats" { return nil } @@ -334,7 +339,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } // check if the client is authenticated - if ctx.RequestContext().Authentication() == nil { + if ctx.Authentication() == nil { // if the client is not authenticated, return an error return &StreamHookError{ HttpError: core.HttpError{ @@ -346,7 +351,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } // check if the client is allowed to subscribe to the stream - clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["readEmployee"] + clientAllowedEntitiesIds, found := ctx.Authentication().Claims()["readEmployee"] if !found { return &StreamHookError{ HttpError: core.HttpError{ From 2766ed7e373a0c741848843cb604cb76a1c83d2c Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:31:16 +0200 Subject: [PATCH 152/173] chore: add test to verify module dependency --- router-tests/modules/stream_publish_test.go | 2 +- .../modules/streams_hooks_combined_test.go | 149 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 router-tests/modules/streams_hooks_combined_test.go diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index a9de5ddcd4..5a36e72f81 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -295,7 +295,7 @@ func TestPublishHook(t *testing.T) { }, func(t *testing.T, xEnv *testenv.Environment) { events.KafkaEnsureTopicExists(t, xEnv, time.Second, "employeeUpdated") resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ - Query: `mutation UpdateEmployeeKafka($employeeID: Int!) { updateEmployeeMyKafka(employeeID: $employeeID, update: {name: "name test"}) { success } }`, + Query: `mutation UpdateEmployeeKafka($employeeID: Int!) { updateEmployeeMyKafka(employeeID: $employeeID, update: {name: "name test"}) { success } }`, Variables: json.RawMessage(`{"employeeID": 3}`), }) require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": true}}}`, resOne.Body) diff --git a/router-tests/modules/streams_hooks_combined_test.go b/router-tests/modules/streams_hooks_combined_test.go new file mode 100644 index 0000000000..a23ab46040 --- /dev/null +++ b/router-tests/modules/streams_hooks_combined_test.go @@ -0,0 +1,149 @@ +package module + +import ( + "encoding/json" + "testing" + "time" + + "github.com/hasura/go-graphql-client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wundergraph/cosmo/router-tests/events" + stream_batch "github.com/wundergraph/cosmo/router-tests/modules/stream-batch" + stream_publish "github.com/wundergraph/cosmo/router-tests/modules/stream-publish" + "github.com/wundergraph/cosmo/router-tests/testenv" + "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/config" + "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" + "go.uber.org/zap/zapcore" +) + +func TestStreamsHooksCombined(t *testing.T) { + t.Parallel() + + t.Run("Test kafka modules can depend on each other", func(t *testing.T) { + t.Parallel() + + type event struct { + data []byte + err error + } + + const Timeout = time.Second * 10 + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "streamBatchModule": stream_batch.StreamBatchModule{ + Callback: func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + + if string(evt.Headers["x-publishModule"]) == "i_was_here" { + evt.Data = []byte(`{"__typename":"Employee","id": 2,"update":{"name":"irrelevant"}}`) + } + } + + return events, nil + }, + }, + "publishModule": stream_publish.PublishModule{ + Callback: func(ctx core.StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + if ctx.PublishEventConfiguration().RootFieldName() != "updateEmployeeMyKafka" { + return events, nil + } + + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + evt.Headers["x-publishModule"] = []byte("i_was_here") + } + + return events, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_publish.PublishModule{}, &stream_batch.StreamBatchModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) + + // start a subscriber + var subscriptionPayload struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionEventsChan := make(chan event) + subscriptionID, err := client.Subscribe(&subscriptionPayload, nil, func(dataValue []byte, errValue error) error { + subscriptionEventsChan <- event{ + data: dataValue, + err: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionID) + + clientRunChan := make(chan error) + go func() { + clientRunChan <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + // publish a message to broker via mutation + // and let publish hook modify the message + resOne := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{ + Query: `mutation UpdateEmployeeKafka($employeeID: Int!) { updateEmployeeMyKafka(employeeID: $employeeID, update: {name: "name test"}) { success } }`, + Variables: json.RawMessage(`{"employeeID": 3}`), + }) + require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": true}}}`, resOne.Body) + + requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") + assert.Len(t, requestLog.All(), 1) + + // wait for the message to be received by the subscriber + testenv.AwaitChannelWithT(t, Timeout, subscriptionEventsChan, func(t *testing.T, args event) { + require.NoError(t, args.err) + // verify that the stream batch hook modified the message, + // which it only does if the publish hook was run before it + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":2,"details":{"forename":"Dustin","surname":"Deus"}}}`, string(args.data)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunChan, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog = xEnv.Observer().FilterMessage("Stream Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) +} From ca97bb8641bfb838930c87a8d84aed33c7640d0f Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Thu, 18 Sep 2025 12:23:04 +0200 Subject: [PATCH 153/173] chore: update go.mod to reflect go.sum --- router-tests/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index ed5bfc3f59..337cfcc049 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -25,8 +25,8 @@ require ( github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e - github.com/wundergraph/cosmo/router v0.0.0-20250825095900-31d35e99379a - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250825082842-0efce3cfd008 + github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 From 5627260fca4fc6212dec1bd2e52d7789beeb27f4 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 18 Sep 2025 12:27:25 +0200 Subject: [PATCH 154/173] refactor: rename StreamBatchEventHook to StreamReceiveEventHook and update related contexts and methods --- adr/cosmo-streams-v1.md | 51 ++++++---- .../module.go | 16 ++-- router-tests/modules/stream_publish_test.go | 2 +- ...m_batch_test.go => stream_receive_test.go} | 18 ++-- .../modules/streams_hooks_combined_test.go | 8 +- router/core/router.go | 4 +- router/core/router_config.go | 2 +- router/core/subscriptions_modules.go | 92 ++++++++++++++----- 8 files changed, 127 insertions(+), 66 deletions(-) rename router-tests/modules/{stream-batch => stream-receive}/module.go (53%) rename router-tests/modules/{stream_batch_test.go => stream_receive_test.go} (89%) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 436dafe45b..ccd7fa2ae1 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -21,7 +21,7 @@ The following interfaces will extend the existing logic in the custom modules. These provide additional control over subscriptions by providing hooks, which are invoked during specific events. - `SubscriptionOnStartHandler`: Called once at subscription start. -- `StreamBatchEventHook`: Called each time a batch of events is received from the provider. +- `StreamReceiveEventHook`: Called each time a batch of events is received from the provider. - `StreamPublishEventHook`: Called each time a batch of events is going to be sent to the provider. ```go @@ -48,8 +48,9 @@ type OperationContext interface { // each provider will have its own event type with custom fields // the StreamEvent interface is used to allow the hooks system to be provider-agnostic -// there could be common fields in future, but for now we don't need them -type StreamEvent interface {} +type StreamEvent interface { + GetData() []byte +} // SubscriptionEventConfiguration is the common interface for the subscription event configuration type SubscriptionEventConfiguration interface { @@ -89,29 +90,41 @@ type SubscriptionOnStartHandler interface { SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error } -type StreamBatchEventHookContext interface { - // the request context - RequestContext() RequestContext - // the subscription event configuration +type StreamReceiveEventHookContext interface { + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // SubscriptionEventConfiguration is the subscription event configuration SubscriptionEventConfiguration() SubscriptionEventConfiguration } -type StreamBatchEventHook interface { +type StreamReceiveEventHook interface { // OnStreamEvents is called each time a batch of events is received from the provider // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - OnStreamEvents(ctx StreamBatchEventHookContext, events []StreamEvent) ([]StreamEvent, error) + OnReceiveEvents(ctx StreamReceiveEventHookContext, events []StreamEvent) ([]StreamEvent, error) } type StreamPublishEventHookContext interface { - // the request context - RequestContext() RequestContext - // the publish event configuration + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // PublishEventConfiguration is the publish event configuration PublishEventConfiguration() PublishEventConfiguration } type StreamPublishEventHook interface { // OnPublishEvents is called each time a batch of events is going to be sent to the provider - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + // Returning an error will result in an error being returned and the client will see the mutation failing OnPublishEvents(ctx StreamPublishEventHookContext, events []StreamEvent) ([]StreamEvent, error) } ``` @@ -177,14 +190,14 @@ func init() { type MyModule struct {} -func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { +func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { // check if the provider is nats - if ctx.StreamContext().ProviderType() != pubsub.ProviderTypeNats { + if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil } // check if the provider id is the one expected by the module - if ctx.StreamContext().ProviderID() != "my-nats" { + if ctx.SubscriptionEventConfiguration().ProviderID() != "my-nats" { return events, nil } @@ -195,13 +208,13 @@ func (m *MyModule) OnStreamEvents(ctx StreamBatchEventHookContext, events []core } // check if the client is authenticated - if ctx.RequestContext().Authentication() == nil { + if ctx.Authentication() == nil { // if the client is not authenticated, return no events return events, nil } // check if the client is allowed to subscribe to the stream - clientAllowedEntitiesIds, found := ctx.RequestContext().Authentication().Claims()["allowedEntitiesIds"] + clientAllowedEntitiesIds, found := ctx.Authentication().Claims()["allowedEntitiesIds"] if !found { return events, fmt.Errorf("client is not allowed to subscribe to the stream") } @@ -266,7 +279,7 @@ func (m *MyModule) Module() core.ModuleInfo { // Interface guards var ( - _ core.StreamBatchEventHook = (*MyModule)(nil) + _ core.StreamReceiveEventHook = (*MyModule)(nil) ) ``` diff --git a/router-tests/modules/stream-batch/module.go b/router-tests/modules/stream-receive/module.go similarity index 53% rename from router-tests/modules/stream-batch/module.go rename to router-tests/modules/stream-receive/module.go index cd62ec09c0..6961234bae 100644 --- a/router-tests/modules/stream-batch/module.go +++ b/router-tests/modules/stream-receive/module.go @@ -7,21 +7,21 @@ import ( "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" ) -const myModuleID = "streamBatchModule" +const myModuleID = "streamReceiveModule" -type StreamBatchModule struct { +type StreamReceiveModule struct { Logger *zap.Logger - Callback func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + Callback func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } -func (m *StreamBatchModule) Provision(ctx *core.ModuleContext) error { +func (m *StreamReceiveModule) Provision(ctx *core.ModuleContext) error { // Assign the logger to the module for non-request related logging m.Logger = ctx.Logger return nil } -func (m *StreamBatchModule) OnStreamEvents(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { +func (m *StreamReceiveModule) OnReceiveEvents(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { m.Logger.Info("Stream Hook has been run") if m.Callback != nil { @@ -31,19 +31,19 @@ func (m *StreamBatchModule) OnStreamEvents(ctx core.StreamBatchEventHookContext, return events, nil } -func (m *StreamBatchModule) Module() core.ModuleInfo { +func (m *StreamReceiveModule) Module() core.ModuleInfo { return core.ModuleInfo{ // This is the ID of your module, it must be unique ID: myModuleID, // The priority of your module, lower numbers are executed first Priority: 1, New: func() core.Module { - return &StreamBatchModule{} + return &StreamReceiveModule{} }, } } // Interface guard var ( - _ core.StreamBatchEventHook = (*StreamBatchModule)(nil) + _ core.StreamReceiveEventHook = (*StreamReceiveModule)(nil) ) diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 5a36e72f81..f83b5a61d3 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -261,7 +261,7 @@ func TestPublishHook(t *testing.T) { return events, nil } - employeeID := ctx.RequestContext().Operation().Variables().GetInt("employeeID") + employeeID := ctx.Operation().Variables().GetInt("employeeID") newEvents := []datasource.StreamEvent{} for _, event := range events { diff --git a/router-tests/modules/stream_batch_test.go b/router-tests/modules/stream_receive_test.go similarity index 89% rename from router-tests/modules/stream_batch_test.go rename to router-tests/modules/stream_receive_test.go index deeda4fb18..66e76e3740 100644 --- a/router-tests/modules/stream_batch_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router-tests/events" - stream_batch "github.com/wundergraph/cosmo/router-tests/modules/stream-batch" + stream_receive "github.com/wundergraph/cosmo/router-tests/modules/stream-receive" "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/core" "github.com/wundergraph/cosmo/router/pkg/config" @@ -18,7 +18,7 @@ import ( "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) -func TestBatchHook(t *testing.T) { +func TestReceiveHook(t *testing.T) { t.Parallel() const Timeout = time.Second * 10 @@ -28,13 +28,13 @@ func TestBatchHook(t *testing.T) { errValue error } - t.Run("Test Batch hook is called", func(t *testing.T) { + t.Run("Test Receive hook is called", func(t *testing.T) { t.Parallel() cfg := config.Config{ Graph: config.Graph{}, Modules: map[string]interface{}{ - "batchModule": stream_batch.StreamBatchModule{}, + "streamReceiveModule": stream_receive.StreamReceiveModule{}, }, } @@ -43,7 +43,7 @@ func TestBatchHook(t *testing.T) { EnableKafka: true, RouterOptions: []core.Option{ core.WithModulesConfig(cfg.Modules), - core.WithCustomModules(&stream_batch.StreamBatchModule{}), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), }, LogObservation: testenv.LogObservationConfig{ Enabled: true, @@ -101,14 +101,14 @@ func TestBatchHook(t *testing.T) { }) }) - t.Run("Test Batch hook could change events", func(t *testing.T) { + t.Run("Test Receive hook could change events", func(t *testing.T) { t.Parallel() cfg := config.Config{ Graph: config.Graph{}, Modules: map[string]interface{}{ - "streamBatchModule": stream_batch.StreamBatchModule{ - Callback: func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + "streamReceiveModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { for _, event := range events { evt, ok := event.(*kafka.Event) if !ok { @@ -128,7 +128,7 @@ func TestBatchHook(t *testing.T) { EnableKafka: true, RouterOptions: []core.Option{ core.WithModulesConfig(cfg.Modules), - core.WithCustomModules(&stream_batch.StreamBatchModule{}), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), }, LogObservation: testenv.LogObservationConfig{ Enabled: true, diff --git a/router-tests/modules/streams_hooks_combined_test.go b/router-tests/modules/streams_hooks_combined_test.go index a23ab46040..3796808a35 100644 --- a/router-tests/modules/streams_hooks_combined_test.go +++ b/router-tests/modules/streams_hooks_combined_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router-tests/events" - stream_batch "github.com/wundergraph/cosmo/router-tests/modules/stream-batch" stream_publish "github.com/wundergraph/cosmo/router-tests/modules/stream-publish" + stream_receive "github.com/wundergraph/cosmo/router-tests/modules/stream-receive" "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/core" "github.com/wundergraph/cosmo/router/pkg/config" @@ -35,8 +35,8 @@ func TestStreamsHooksCombined(t *testing.T) { cfg := config.Config{ Graph: config.Graph{}, Modules: map[string]interface{}{ - "streamBatchModule": stream_batch.StreamBatchModule{ - Callback: func(ctx core.StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + "streamReceiveModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { for _, event := range events { evt, ok := event.(*kafka.Event) if !ok { @@ -76,7 +76,7 @@ func TestStreamsHooksCombined(t *testing.T) { EnableKafka: true, RouterOptions: []core.Option{ core.WithModulesConfig(cfg.Modules), - core.WithCustomModules(&stream_publish.PublishModule{}, &stream_batch.StreamBatchModule{}), + core.WithCustomModules(&stream_publish.PublishModule{}, &stream_receive.StreamReceiveModule{}), }, LogObservation: testenv.LogObservationConfig{ Enabled: true, diff --git a/router/core/router.go b/router/core/router.go index 6bdee85f88..f68939dccd 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -650,8 +650,8 @@ func (r *Router) initModules(ctx context.Context) error { r.subscriptionHooks.onPublishEvents = append(r.subscriptionHooks.onPublishEvents, handler.OnPublishEvents) } - if handler, ok := moduleInstance.(StreamBatchEventHook); ok { - r.subscriptionHooks.onStreamEvents = append(r.subscriptionHooks.onStreamEvents, handler.OnStreamEvents) + if handler, ok := moduleInstance.(StreamReceiveEventHook); ok { + r.subscriptionHooks.onStreamEvents = append(r.subscriptionHooks.onStreamEvents, handler.OnReceiveEvents) } r.modules = append(r.modules, moduleInstance) diff --git a/router/core/router_config.go b/router/core/router_config.go index ccfa09e1bb..b7eda43627 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -29,7 +29,7 @@ import ( type subscriptionHooks struct { onStart []func(ctx SubscriptionOnStartHookContext) error onPublishEvents []func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) - onStreamEvents []func(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + onStreamEvents []func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index d76ffa38a7..ce4c981279 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -64,12 +64,27 @@ type SubscriptionOnStartHookContext interface { } type pubSubPublishEventHookContext struct { - requestContext RequestContext + request *http.Request + logger *zap.Logger + operation OperationContext + authentication authentication.Authentication publishEventConfiguration datasource.PublishEventConfiguration } -func (c *pubSubPublishEventHookContext) RequestContext() RequestContext { - return c.requestContext +func (c *pubSubPublishEventHookContext) Request() *http.Request { + return c.request +} + +func (c *pubSubPublishEventHookContext) Logger() *zap.Logger { + return c.logger +} + +func (c *pubSubPublishEventHookContext) Operation() OperationContext { + return c.operation +} + +func (c *pubSubPublishEventHookContext) Authentication() authentication.Authentication { + return c.authentication } func (c *pubSubPublishEventHookContext) PublishEventConfiguration() datasource.PublishEventConfiguration { @@ -201,23 +216,35 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext } } -type StreamBatchEventHookContext interface { - // the request context - RequestContext() RequestContext - // the subscription event configuration +type StreamReceiveEventHookContext interface { + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // SubscriptionEventConfiguration the subscription event configuration SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration } -type StreamBatchEventHook interface { - // OnStreamEvents is called each time a batch of events is received from the provider +type StreamReceiveEventHook interface { + // OnReceiveEvents is called each time a batch of events is received from the provider // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - OnStreamEvents(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + OnReceiveEvents(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } type StreamPublishEventHookContext interface { - // the request context - RequestContext() RequestContext - // the publish event configuration + // Request is the original request received by the router. + Request() *http.Request + // Logger is the logger for the request + Logger() *zap.Logger + // Operation is the GraphQL operation + Operation() OperationContext + // Authentication is the authentication for the request + Authentication() authentication.Authentication + // PublishEventConfiguration the publish event configuration PublishEventConfiguration() datasource.PublishEventConfiguration } @@ -235,7 +262,10 @@ func NewPubSubOnPublishEventsHook(fn func(ctx StreamPublishEventHookContext, eve return func(ctx context.Context, pubConf datasource.PublishEventConfiguration, evts []datasource.StreamEvent) ([]datasource.StreamEvent, error) { requestContext := getRequestContext(ctx) hookCtx := &pubSubPublishEventHookContext{ - requestContext: requestContext, + request: requestContext.Request(), + logger: requestContext.Logger(), + operation: requestContext.Operation(), + authentication: requestContext.Authentication(), publishEventConfiguration: pubConf, } @@ -243,28 +273,46 @@ func NewPubSubOnPublishEventsHook(fn func(ctx StreamPublishEventHookContext, eve } } -type pubSubStreamBatchEventHookContext struct { - requestContext RequestContext +type pubSubStreamReceiveEventHookContext struct { + request *http.Request + logger *zap.Logger + operation OperationContext + authentication authentication.Authentication subscriptionEventConfiguration datasource.SubscriptionEventConfiguration } -func (c *pubSubStreamBatchEventHookContext) RequestContext() RequestContext { - return c.requestContext +func (c *pubSubStreamReceiveEventHookContext) Request() *http.Request { + return c.request +} + +func (c *pubSubStreamReceiveEventHookContext) Logger() *zap.Logger { + return c.logger +} + +func (c *pubSubStreamReceiveEventHookContext) Operation() OperationContext { + return c.operation } -func (c *pubSubStreamBatchEventHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { +func (c *pubSubStreamReceiveEventHookContext) Authentication() authentication.Authentication { + return c.authentication +} + +func (c *pubSubStreamReceiveEventHookContext) SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration { return c.subscriptionEventConfiguration } -func NewPubSubOnStreamEventsHook(fn func(ctx StreamBatchEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnStreamEventsFn { +func NewPubSubOnStreamEventsHook(fn func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnStreamEventsFn { if fn == nil { return nil } return func(ctx context.Context, subConf datasource.SubscriptionEventConfiguration, evts []datasource.StreamEvent) ([]datasource.StreamEvent, error) { requestContext := getRequestContext(ctx) - hookCtx := &pubSubStreamBatchEventHookContext{ - requestContext: requestContext, + hookCtx := &pubSubStreamReceiveEventHookContext{ + request: requestContext.Request(), + logger: requestContext.Logger(), + operation: requestContext.Operation(), + authentication: requestContext.Authentication(), subscriptionEventConfiguration: subConf, } From 8d60929bec45511a69a490701e466fcb84c291c5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 18 Sep 2025 12:42:20 +0200 Subject: [PATCH 155/173] refactor: rename OnStreamEvents to OnReceiveEvents and update related hooks and methods --- router-tests/modules/stream_receive_test.go | 121 ++++++++++++++++++ router/core/factoryresolver.go | 8 +- router/core/router.go | 2 +- router/core/router_config.go | 2 +- router/core/subscriptions_modules.go | 2 +- router/pkg/pubsub/datasource/hooks.go | 6 +- .../datasource/subscription_event_updater.go | 6 +- .../subscription_event_updater_test.go | 36 +++--- router/pkg/pubsub/pubsub_test.go | 4 +- 9 files changed, 153 insertions(+), 34 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 66e76e3740..21d469c16e 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -1,6 +1,7 @@ package module_test import ( + "net/http" "testing" "time" @@ -185,4 +186,124 @@ func TestReceiveHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) + + t.Run("Test Receive hook change events of one of multiple subscriptions", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "streamReceiveModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + if hdr, ok := ctx.Request().Header["x-custom-header"]; ok && hdr[0] == "dont-change" { + return events, nil + } + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + evt.Data = []byte(`{"__typename":"Employee","id": 3,"update":{"name":"foo"}}`) + } + + return events, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + client2 := graphql.NewSubscriptionClient(surl) + client2.WithWebSocketOptions(graphql.WebsocketOptions{ + HTTPHeader: http.Header{ + "x-custom-header": []string{"dont-change"}, + }, + }) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + subscriptionArgsCh2 := make(chan kafkaSubscriptionArgs) + subscriptionTwoID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh2 <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionTwoID) + + clientRunCh2 := make(chan error) + go func() { + clientRunCh2 <- client2.Run() + }() + + xEnv.WaitForSubscriptionCount(2, Timeout) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":3,"details":{"forename":"Stefan","surname":"Avram"}}}`, string(args.dataValue)) + }) + + testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh2, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) + }) + + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + require.NoError(t, client2.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("Stream Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) } diff --git a/router/core/factoryresolver.go b/router/core/factoryresolver.go index eacd8b034f..e452c2cf90 100644 --- a/router/core/factoryresolver.go +++ b/router/core/factoryresolver.go @@ -484,9 +484,9 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod onPublishEventsFns[i] = NewPubSubOnPublishEventsHook(fn) } - onStreamEventsFns := make([]pubsub_datasource.OnStreamEventsFn, len(l.subscriptionHooks.onStreamEvents)) - for i, fn := range l.subscriptionHooks.onStreamEvents { - onStreamEventsFns[i] = NewPubSubOnStreamEventsHook(fn) + onReceiveEventsFns := make([]pubsub_datasource.OnReceiveEventsFn, len(l.subscriptionHooks.onReceiveEvents)) + for i, fn := range l.subscriptionHooks.onReceiveEvents { + onReceiveEventsFns[i] = NewPubSubOnReceiveEventsHook(fn) } factoryProviders, factoryDataSources, err := pubsub.BuildProvidersAndDataSources( @@ -498,7 +498,7 @@ func (l *Loader) Load(engineConfig *nodev1.EngineConfiguration, subgraphs []*nod l.resolver.InstanceData().ListenAddress, pubsub_datasource.Hooks{ SubscriptionOnStart: subscriptionOnStartFns, - OnStreamEvents: onStreamEventsFns, + OnReceiveEvents: onReceiveEventsFns, OnPublishEvents: onPublishEventsFns, }, ) diff --git a/router/core/router.go b/router/core/router.go index f68939dccd..8955d14485 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -651,7 +651,7 @@ func (r *Router) initModules(ctx context.Context) error { } if handler, ok := moduleInstance.(StreamReceiveEventHook); ok { - r.subscriptionHooks.onStreamEvents = append(r.subscriptionHooks.onStreamEvents, handler.OnReceiveEvents) + r.subscriptionHooks.onReceiveEvents = append(r.subscriptionHooks.onReceiveEvents, handler.OnReceiveEvents) } r.modules = append(r.modules, moduleInstance) diff --git a/router/core/router_config.go b/router/core/router_config.go index b7eda43627..78d08c0c8a 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -29,7 +29,7 @@ import ( type subscriptionHooks struct { onStart []func(ctx SubscriptionOnStartHookContext) error onPublishEvents []func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) - onStreamEvents []func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + onReceiveEvents []func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index ce4c981279..0ec7056d88 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -301,7 +301,7 @@ func (c *pubSubStreamReceiveEventHookContext) SubscriptionEventConfiguration() d return c.subscriptionEventConfiguration } -func NewPubSubOnStreamEventsHook(fn func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnStreamEventsFn { +func NewPubSubOnReceiveEventsHook(fn func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnReceiveEventsFn { if fn == nil { return nil } diff --git a/router/pkg/pubsub/datasource/hooks.go b/router/pkg/pubsub/datasource/hooks.go index 1ffa89e1f1..abab8b8ef1 100644 --- a/router/pkg/pubsub/datasource/hooks.go +++ b/router/pkg/pubsub/datasource/hooks.go @@ -6,15 +6,15 @@ import ( "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) -type SubscriptionOnStartFn func(ctx resolve.StartupHookContext, subConf SubscriptionEventConfiguration) (error) +type SubscriptionOnStartFn func(ctx resolve.StartupHookContext, subConf SubscriptionEventConfiguration) error type OnPublishEventsFn func(ctx context.Context, pubConf PublishEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) -type OnStreamEventsFn func(ctx context.Context, subConf SubscriptionEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) +type OnReceiveEventsFn func(ctx context.Context, subConf SubscriptionEventConfiguration, evts []StreamEvent) ([]StreamEvent, error) // Hooks contains hooks for the pubsub providers and data sources type Hooks struct { SubscriptionOnStart []SubscriptionOnStartFn - OnStreamEvents []OnStreamEventsFn + OnReceiveEvents []OnReceiveEventsFn OnPublishEvents []OnPublishEventsFn } diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 22d5b2438c..7fc0fe0725 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -32,12 +32,12 @@ func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { } func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { - if len(s.hooks.OnStreamEvents) == 0 { + if len(s.hooks.OnReceiveEvents) == 0 { s.updateEvents(events) return nil } - processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnStreamEvents) + processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnReceiveEvents) // updates the events even if the hooks fail // if a hook doesn't want to send the events, it should return no events! s.updateEvents(processedEvents) @@ -83,7 +83,7 @@ func applyStreamEventHooks( ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent, - hooks []OnStreamEventsFn) ([]StreamEvent, error) { + hooks []OnReceiveEventsFn) ([]StreamEvent, error) { currentEvents := events for _, hook := range hooks { var err error diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go index 96be85e9f8..fc32cb33a0 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater_test.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -95,7 +95,7 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Success(t *testing.T) { ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } @@ -136,7 +136,7 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error(t *testing.T) { ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } @@ -181,7 +181,7 @@ func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ - OnStreamEvents: []OnStreamEventsFn{hook1, hook2}, + OnReceiveEvents: []OnReceiveEventsFn{hook1, hook2}, }, } @@ -263,7 +263,7 @@ func TestSubscriptionEventUpdater_SetHooks(t *testing.T) { } hooks := Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, } updater := &subscriptionEventUpdater{ @@ -292,7 +292,7 @@ func TestNewSubscriptionEventUpdater(t *testing.T) { } hooks := Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, } updater := NewSubscriptionEventUpdater(ctx, config, hooks, mockUpdater, zap.NewNop()) @@ -320,7 +320,7 @@ func TestApplyStreamEventHooks_NoHooks(t *testing.T) { &testEvent{data: []byte("test data")}, } - result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{}) + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnReceiveEventsFn{}) assert.NoError(t, err) assert.Equal(t, originalEvents, result) @@ -344,7 +344,7 @@ func TestApplyStreamEventHooks_SingleHook_Success(t *testing.T) { return modifiedEvents, nil } - result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook}) + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnReceiveEventsFn{hook}) assert.NoError(t, err) assert.Equal(t, modifiedEvents, result) @@ -366,7 +366,7 @@ func TestApplyStreamEventHooks_SingleHook_Error(t *testing.T) { return nil, hookError } - result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook}) + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnReceiveEventsFn{hook}) assert.Error(t, err) assert.Equal(t, hookError, err) @@ -400,7 +400,7 @@ func TestApplyStreamEventHooks_MultipleHooks_Success(t *testing.T) { return []StreamEvent{&testEvent{data: []byte("final")}}, nil } - result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook1, hook2, hook3}) + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnReceiveEventsFn{hook1, hook2, hook3}) select { case receivedArgs1 := <-receivedArgs1: @@ -459,7 +459,7 @@ func TestApplyStreamEventHooks_MultipleHooks_MiddleHookError(t *testing.T) { return []StreamEvent{&testEvent{data: []byte("final")}}, nil } - result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnStreamEventsFn{hook1, hook2, hook3}) + result, err := applyStreamEventHooks(ctx, config, originalEvents, []OnReceiveEventsFn{hook1, hook2, hook3}) assert.Error(t, err) assert.Equal(t, middleHookError, err) @@ -558,7 +558,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t // Create a mock StreamHookError with CloseSubscription=true mockHookError := &mockStreamHookError{ closeSubscription: true, - message: "subscription should close", + message: "subscription should close", } // Define hook that returns a StreamHookError with CloseSubscription=true @@ -571,7 +571,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } @@ -598,7 +598,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription // Create a mock StreamHookError with CloseSubscription=false mockHookError := &mockStreamHookError{ closeSubscription: false, - message: "subscription should not close", + message: "subscription should not close", } // Define hook that returns a StreamHookError with CloseSubscription=false @@ -611,7 +611,7 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } @@ -648,7 +648,7 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *te // Test with a real zap logger to verify error logging behavior // The logger.Error() call should be executed when an error occurs updater := NewSubscriptionEventUpdater(ctx, config, Hooks{ - OnStreamEvents: []OnStreamEventsFn{testHook}, + OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, mockUpdater, logger) err := updater.Update(events) @@ -657,7 +657,7 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *te assert.NoError(t, err) // Assert that Update was not called on the eventUpdater mockUpdater.AssertNotCalled(t, "Update") - + msgs := logObserver.FilterMessageSnippet("An error occurred while processing stream events hooks").TakeAll() assert.Equal(t, 1, len(msgs)) } @@ -665,7 +665,7 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *te // mockStreamHookError implements the CloseSubscription() method for testing type mockStreamHookError struct { closeSubscription bool - message string + message string } func (e *mockStreamHookError) Error() string { @@ -675,5 +675,3 @@ func (e *mockStreamHookError) Error() string { func (e *mockStreamHookError) CloseSubscription() bool { return e.closeSubscription } - - diff --git a/router/pkg/pubsub/pubsub_test.go b/router/pkg/pubsub/pubsub_test.go index 74d993a050..768ec7f476 100644 --- a/router/pkg/pubsub/pubsub_test.go +++ b/router/pkg/pubsub/pubsub_test.go @@ -60,7 +60,7 @@ func TestBuild_OK(t *testing.T) { mockPubSubProvider.On("ID").Return("provider-1") mockPubSubProvider.On("SetHooks", datasource.Hooks{ - OnStreamEvents: []datasource.OnStreamEventsFn(nil), + OnReceiveEvents: []datasource.OnReceiveEventsFn(nil), OnPublishEvents: []datasource.OnPublishEventsFn(nil), }).Return(nil) @@ -239,7 +239,7 @@ func TestBuild_ShouldNotInitializeProviderIfNotUsed(t *testing.T) { mockPubSubUsedProvider.On("ID").Return("provider-2") mockPubSubUsedProvider.On("SetHooks", datasource.Hooks{ - OnStreamEvents: []datasource.OnStreamEventsFn(nil), + OnReceiveEvents: []datasource.OnReceiveEventsFn(nil), OnPublishEvents: []datasource.OnPublishEventsFn(nil), }).Return(nil) From f6fa44cee1c1d5acc47f730d00fa8e44349616a3 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 18 Sep 2025 13:02:12 +0200 Subject: [PATCH 156/173] fix: use CanonicalHeaderKey for x-custom-header in streamReceiveModule and subscription client --- router-tests/modules/stream_receive_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 21d469c16e..1c05fe30fd 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -195,7 +195,7 @@ func TestReceiveHook(t *testing.T) { Modules: map[string]interface{}{ "streamReceiveModule": stream_receive.StreamReceiveModule{ Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - if hdr, ok := ctx.Request().Header["x-custom-header"]; ok && hdr[0] == "dont-change" { + if hdr, ok := ctx.Request().Header[http.CanonicalHeaderKey("x-custom-header")]; ok && hdr[0] == "dont-change" { return events, nil } for _, event := range events { @@ -242,7 +242,7 @@ func TestReceiveHook(t *testing.T) { client2 := graphql.NewSubscriptionClient(surl) client2.WithWebSocketOptions(graphql.WebsocketOptions{ HTTPHeader: http.Header{ - "x-custom-header": []string{"dont-change"}, + http.CanonicalHeaderKey("x-custom-header"): []string{"dont-change"}, }, }) From 3cdb8178797c03f892320133dee620ee1c6de8f7 Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:17:13 +0200 Subject: [PATCH 157/173] chore: add test for subscription closing via hook --- router-tests/modules/stream_receive_test.go | 83 +++++++++++++++++++++ router/demo.config.yaml | 8 +- router/go.mod | 2 +- router/pkg/pubsub/kafka/adapter.go | 9 ++- 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 1c05fe30fd..8f8e04c52a 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -1,6 +1,7 @@ package module_test import ( + "errors" "net/http" "testing" "time" @@ -19,6 +20,18 @@ import ( "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) +type errorWithCloseSubscription struct { + err error +} + +func (e *errorWithCloseSubscription) Error() string { + return e.err.Error() +} + +func (e *errorWithCloseSubscription) CloseSubscription() bool { + return true +} + func TestReceiveHook(t *testing.T) { t.Parallel() @@ -306,4 +319,74 @@ func TestReceiveHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) + + t.Run("Test Batch hook can close subscriptions", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "streamBatchModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + return nil, &errorWithCloseSubscription{err: errors.New("test error from streamevents hook")} + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + xEnv.WaitForSubscriptionCount(0, Timeout) + + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "client should have completed when server closed connection") + }) + }) } diff --git a/router/demo.config.yaml b/router/demo.config.yaml index 9a72e31de2..d2a41d59d9 100644 --- a/router/demo.config.yaml +++ b/router/demo.config.yaml @@ -19,4 +19,10 @@ events: redis: - id: my-redis urls: - - "redis://localhost:6379/2" \ No newline at end of file + - "redis://localhost:6379/2" + +log_level: info + +persisted_operations: + safelist: + enabled: false \ No newline at end of file diff --git a/router/go.mod b/router/go.mod index 7ac0e475b7..af75abff8c 100644 --- a/router/go.mod +++ b/router/go.mod @@ -162,4 +162,4 @@ require ( // Remember you can use Go workspaces to avoid using replace directives in multiple go.mod files // Use what is best for your personal workflow. See CONTRIBUTING.md for more information -// replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 +replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index a5b63d7b26..e454e4100f 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -57,6 +57,7 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u // If the context was canceled, the error is wrapped in a fetch error if errors.Is(fetchError.Err, context.Canceled) { + fmt.Println("GOT ERROR") return fetchError.Err } @@ -77,6 +78,8 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u } } + fmt.Println("GOT NO ERROR") + iter := fetches.RecordIter() for !iter.Done() { r := iter.Next() @@ -137,7 +140,11 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri go func() { - defer p.closeWg.Done() + defer func() { + log.Warn("-----> poller done") + p.closeWg.Done() + p.Shutdown(ctx) + }() err := p.topicPoller(ctx, client, updater) if err != nil { From 6dbbe94cccf444da5b50723ec0fa247daea09a09 Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Thu, 18 Sep 2025 16:59:58 +0200 Subject: [PATCH 158/173] Revert "chore: add test for subscription closing via hook" This reverts commit 3cdb8178797c03f892320133dee620ee1c6de8f7. --- router-tests/modules/stream_receive_test.go | 83 --------------------- router/demo.config.yaml | 8 +- router/go.mod | 2 +- router/pkg/pubsub/kafka/adapter.go | 9 +-- 4 files changed, 3 insertions(+), 99 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 8f8e04c52a..1c05fe30fd 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -1,7 +1,6 @@ package module_test import ( - "errors" "net/http" "testing" "time" @@ -20,18 +19,6 @@ import ( "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) -type errorWithCloseSubscription struct { - err error -} - -func (e *errorWithCloseSubscription) Error() string { - return e.err.Error() -} - -func (e *errorWithCloseSubscription) CloseSubscription() bool { - return true -} - func TestReceiveHook(t *testing.T) { t.Parallel() @@ -319,74 +306,4 @@ func TestReceiveHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) - - t.Run("Test Batch hook can close subscriptions", func(t *testing.T) { - t.Parallel() - - cfg := config.Config{ - Graph: config.Graph{}, - Modules: map[string]interface{}{ - "streamBatchModule": stream_receive.StreamReceiveModule{ - Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - return nil, &errorWithCloseSubscription{err: errors.New("test error from streamevents hook")} - }, - }, - }, - } - - testenv.Run(t, &testenv.Config{ - RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, - EnableKafka: true, - RouterOptions: []core.Option{ - core.WithModulesConfig(cfg.Modules), - core.WithCustomModules(&stream_receive.StreamReceiveModule{}), - }, - LogObservation: testenv.LogObservationConfig{ - Enabled: true, - LogLevel: zapcore.InfoLevel, - }, - }, func(t *testing.T, xEnv *testenv.Environment) { - topics := []string{"employeeUpdated"} - events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) - - var subscriptionOne struct { - employeeUpdatedMyKafka struct { - ID float64 `graphql:"id"` - Details struct { - Forename string `graphql:"forename"` - Surname string `graphql:"surname"` - } `graphql:"details"` - } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` - } - - surl := xEnv.GraphQLWebSocketSubscriptionURL() - client := graphql.NewSubscriptionClient(surl) - - subscriptionArgsCh := make(chan kafkaSubscriptionArgs) - subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { - subscriptionArgsCh <- kafkaSubscriptionArgs{ - dataValue: dataValue, - errValue: errValue, - } - return nil - }) - require.NoError(t, err) - require.NotEmpty(t, subscriptionOneID) - - clientRunCh := make(chan error) - go func() { - clientRunCh <- client.Run() - }() - - xEnv.WaitForSubscriptionCount(1, Timeout) - - events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) - - xEnv.WaitForSubscriptionCount(0, Timeout) - - testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { - require.NoError(t, err) - }, "client should have completed when server closed connection") - }) - }) } diff --git a/router/demo.config.yaml b/router/demo.config.yaml index d2a41d59d9..9a72e31de2 100644 --- a/router/demo.config.yaml +++ b/router/demo.config.yaml @@ -19,10 +19,4 @@ events: redis: - id: my-redis urls: - - "redis://localhost:6379/2" - -log_level: info - -persisted_operations: - safelist: - enabled: false \ No newline at end of file + - "redis://localhost:6379/2" \ No newline at end of file diff --git a/router/go.mod b/router/go.mod index af75abff8c..7ac0e475b7 100644 --- a/router/go.mod +++ b/router/go.mod @@ -162,4 +162,4 @@ require ( // Remember you can use Go workspaces to avoid using replace directives in multiple go.mod files // Use what is best for your personal workflow. See CONTRIBUTING.md for more information -replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 +// replace github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index e454e4100f..a5b63d7b26 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -57,7 +57,6 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u // If the context was canceled, the error is wrapped in a fetch error if errors.Is(fetchError.Err, context.Canceled) { - fmt.Println("GOT ERROR") return fetchError.Err } @@ -78,8 +77,6 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u } } - fmt.Println("GOT NO ERROR") - iter := fetches.RecordIter() for !iter.Done() { r := iter.Next() @@ -140,11 +137,7 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri go func() { - defer func() { - log.Warn("-----> poller done") - p.closeWg.Done() - p.Shutdown(ctx) - }() + defer p.closeWg.Done() err := p.topicPoller(ctx, client, updater) if err != nil { From f945f973b083e1b9429711f79fc935732b7d10ed Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Thu, 18 Sep 2025 17:04:07 +0200 Subject: [PATCH 159/173] chore: add test to verify subscription close via hook --- router-tests/modules/stream_receive_test.go | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 1c05fe30fd..5bb1ea445a 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -1,6 +1,8 @@ package module_test import ( + "errors" + "fmt" "net/http" "testing" "time" @@ -19,6 +21,18 @@ import ( "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" ) +type errorWithCloseSubscription struct { + err error +} + +func (e *errorWithCloseSubscription) Error() string { + return e.err.Error() +} + +func (e *errorWithCloseSubscription) CloseSubscription() bool { + return true +} + func TestReceiveHook(t *testing.T) { t.Parallel() @@ -306,4 +320,78 @@ func TestReceiveHook(t *testing.T) { assert.Len(t, requestLog.All(), 1) }) }) + + t.Run("Test Batch hook can close Kafka subscriptions", func(t *testing.T) { + t.Parallel() + + cfg := config.Config{ + Graph: config.Graph{}, + Modules: map[string]interface{}{ + "streamBatchModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + return nil, &errorWithCloseSubscription{err: errors.New("test error from streamevents hook")} + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + fmt.Println(client.GetSubscription(subscriptionOneID).GetStatus()) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + // Wait for server to close the subscription connection + xEnv.WaitForSubscriptionCount(0, Timeout) + + // Verify that client.Run() completed when server closed the connection + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "client should have completed when server closed connection") + }) + }) } From 8e87e78e45e922602a763e523145e5013842e148 Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Mon, 22 Sep 2025 16:39:11 +0200 Subject: [PATCH 160/173] fix: close kafka clients + subscribers on topic poller exit --- router-tests/events/utils.go | 2 +- router-tests/modules/stream_receive_test.go | 13 +++++++------ router/pkg/pubsub/kafka/adapter.go | 7 ++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/router-tests/events/utils.go b/router-tests/events/utils.go index 03bac2d7d3..95705efb58 100644 --- a/router-tests/events/utils.go +++ b/router-tests/events/utils.go @@ -118,4 +118,4 @@ func ReadRedisMessages(t *testing.T, xEnv *testenv.Environment, channelName stri }) return sub.Channel(), nil -} \ No newline at end of file +} diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 5bb1ea445a..66bc40f11b 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -2,13 +2,10 @@ package module_test import ( "errors" - "fmt" "net/http" "testing" "time" - "go.uber.org/zap/zapcore" - "github.com/hasura/go-graphql-client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,6 +16,7 @@ import ( "github.com/wundergraph/cosmo/router/pkg/config" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" + "go.uber.org/zap/zapcore" ) type errorWithCloseSubscription struct { @@ -321,7 +319,7 @@ func TestReceiveHook(t *testing.T) { }) }) - t.Run("Test Batch hook can close Kafka subscriptions", func(t *testing.T) { + t.Run("Test Batch hook error should close Kafka clients and subscriptions", func(t *testing.T) { t.Parallel() cfg := config.Config{ @@ -381,8 +379,6 @@ func TestReceiveHook(t *testing.T) { xEnv.WaitForSubscriptionCount(1, Timeout) - fmt.Println(client.GetSubscription(subscriptionOneID).GetStatus()) - events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) // Wait for server to close the subscription connection @@ -392,6 +388,11 @@ func TestReceiveHook(t *testing.T) { testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { require.NoError(t, err) }, "client should have completed when server closed connection") + + // Verify that Kafka connections are also closed by checking for "poller canceled" log messages + // The Kafka adapter logs this when connections are closed due to context cancellation + kafkaPollerLogs := xEnv.Observer().FilterMessage("poller error") + assert.GreaterOrEqual(t, len(kafkaPollerLogs.All()), 1, "Expected at least one Kafka poller to be canceled") }) }) } diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index a5b63d7b26..d4395651be 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -11,6 +11,7 @@ import ( "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kgo" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" + "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" "go.uber.org/zap" ) @@ -137,7 +138,11 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri go func() { - defer p.closeWg.Done() + defer func() { + client.Close() + updater.Close(resolve.SubscriptionCloseKindNormal) + p.closeWg.Done() + }() err := p.topicPoller(ctx, client, updater) if err != nil { From 6ce23a0f85406c52da987810e28d49b6f04b9af5 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 11:59:38 +0200 Subject: [PATCH 161/173] chore: now SubscriptionEventUpdater runs hooks separately for each active subscription --- router-tests/modules/stream_receive_test.go | 57 +++++-- router/core/subscriptions_modules.go | 5 + router/pkg/pubsub/datasource/mocks.go | 25 +--- router/pkg/pubsub/datasource/mocks_resolve.go | 140 ++++++++++++++++++ router/pkg/pubsub/datasource/provider.go | 1 + .../pubsub/datasource/pubsubprovider_test.go | 5 + .../datasource/subscription_datasource.go | 2 +- .../datasource/subscription_event_updater.go | 80 +++++----- .../subscription_event_updater_test.go | 122 +++++++-------- router/pkg/pubsub/kafka/adapter.go | 6 +- router/pkg/pubsub/kafka/engine_datasource.go | 7 +- router/pkg/pubsub/nats/adapter.go | 35 +---- router/pkg/pubsub/nats/engine_datasource.go | 5 + router/pkg/pubsub/redis/adapter.go | 15 +- router/pkg/pubsub/redis/engine_datasource.go | 5 + 15 files changed, 326 insertions(+), 184 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 66bc40f11b..de2d24d6ae 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -9,13 +9,17 @@ import ( "github.com/hasura/go-graphql-client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + integration "github.com/wundergraph/cosmo/router-tests" "github.com/wundergraph/cosmo/router-tests/events" + "github.com/wundergraph/cosmo/router-tests/jwks" stream_receive "github.com/wundergraph/cosmo/router-tests/modules/stream-receive" "github.com/wundergraph/cosmo/router-tests/testenv" "github.com/wundergraph/cosmo/router/core" + "github.com/wundergraph/cosmo/router/pkg/authentication" "github.com/wundergraph/cosmo/router/pkg/config" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" "github.com/wundergraph/cosmo/router/pkg/pubsub/kafka" + "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -204,10 +208,14 @@ func TestReceiveHook(t *testing.T) { cfg := config.Config{ Graph: config.Graph{}, + Modules: map[string]interface{}{ "streamReceiveModule": stream_receive.StreamReceiveModule{ Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - if hdr, ok := ctx.Request().Header[http.CanonicalHeaderKey("x-custom-header")]; ok && hdr[0] == "dont-change" { + if ctx.Authentication() == nil { + return events, nil + } + if val, ok := ctx.Authentication().Claims()["sub"]; !ok || val != "user-2" { return events, nil } for _, event := range events { @@ -224,12 +232,32 @@ func TestReceiveHook(t *testing.T) { }, } + authServer, err := jwks.NewServer(t) + require.NoError(t, err) + defer authServer.Close() + + JwksName := "my-jwks-server" + + tokenDecoder, _ := authentication.NewJwksTokenDecoder(integration.NewContextWithCancel(t), zap.NewNop(), []authentication.JWKSConfig{{ + URL: authServer.JWKSURL(), + RefreshInterval: time.Second * 5, + }}) + jwksOpts := authentication.HttpHeaderAuthenticatorOptions{ + Name: JwksName, + TokenDecoder: tokenDecoder, + } + + authenticator, err := authentication.NewHttpHeaderAuthenticator(jwksOpts) + require.NoError(t, err) + authenticators := []authentication.Authenticator{authenticator} + testenv.Run(t, &testenv.Config{ RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, EnableKafka: true, RouterOptions: []core.Option{ core.WithModulesConfig(cfg.Modules), core.WithCustomModules(&stream_receive.StreamReceiveModule{}), + core.WithAccessController(core.NewAccessController(authenticators, false)), }, LogObservation: testenv.LogObservationConfig{ Enabled: true, @@ -249,13 +277,20 @@ func TestReceiveHook(t *testing.T) { } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` } + token, err := authServer.Token(map[string]interface{}{ + "sub": "user-2", + }) + require.NoError(t, err) + + headers := http.Header{ + "Authorization": []string{"Bearer " + token}, + } + surl := xEnv.GraphQLWebSocketSubscriptionURL() client := graphql.NewSubscriptionClient(surl) client2 := graphql.NewSubscriptionClient(surl) client2.WithWebSocketOptions(graphql.WebsocketOptions{ - HTTPHeader: http.Header{ - http.CanonicalHeaderKey("x-custom-header"): []string{"dont-change"}, - }, + HTTPHeader: headers, }) subscriptionArgsCh := make(chan kafkaSubscriptionArgs) @@ -275,7 +310,7 @@ func TestReceiveHook(t *testing.T) { }() subscriptionArgsCh2 := make(chan kafkaSubscriptionArgs) - subscriptionTwoID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionTwoID, err := client2.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { subscriptionArgsCh2 <- kafkaSubscriptionArgs{ dataValue: dataValue, errValue: errValue, @@ -296,26 +331,30 @@ func TestReceiveHook(t *testing.T) { testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) - require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":3,"details":{"forename":"Stefan","surname":"Avram"}}}`, string(args.dataValue)) + assert.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) }) testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh2, func(t *testing.T, args kafkaSubscriptionArgs) { require.NoError(t, args.errValue) - require.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":1,"details":{"forename":"Jens","surname":"Neuse"}}}`, string(args.dataValue)) + assert.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":3,"details":{"forename":"Stefan","surname":"Avram"}}}`, string(args.dataValue)) }) + unSub1Err := client.Unsubscribe(subscriptionOneID) + require.NoError(t, unSub1Err) require.NoError(t, client.Close()) testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { require.NoError(t, err) }, "unable to close client before timeout") + unSub2Err := client2.Unsubscribe(subscriptionTwoID) + require.NoError(t, unSub2Err) require.NoError(t, client2.Close()) - testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + testenv.AwaitChannelWithT(t, Timeout, clientRunCh2, func(t *testing.T, err error) { require.NoError(t, err) }, "unable to close client before timeout") requestLog := xEnv.Observer().FilterMessage("Stream Hook has been run") - assert.Len(t, requestLog.All(), 1) + assert.Len(t, requestLog.All(), 2) }) }) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 0ec7056d88..d23bc54b05 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -135,6 +135,11 @@ func (e *EngineEvent) GetData() []byte { return e.Data } +func (e *EngineEvent) Clone() datasource.StreamEvent { + e2 := *e + return &e2 +} + type engineSubscriptionOnStartHookContext struct { request *http.Request logger *zap.Logger diff --git a/router/pkg/pubsub/datasource/mocks.go b/router/pkg/pubsub/datasource/mocks.go index 764c28404e..6b2a910d65 100644 --- a/router/pkg/pubsub/datasource/mocks.go +++ b/router/pkg/pubsub/datasource/mocks.go @@ -1210,20 +1210,9 @@ func (_c *MockSubscriptionEventUpdater_SetHooks_Call) RunAndReturn(run func(hook } // Update provides a mock function for the type MockSubscriptionEventUpdater -func (_mock *MockSubscriptionEventUpdater) Update(events []StreamEvent) error { - ret := _mock.Called(events) - - if len(ret) == 0 { - panic("no return value specified for Update") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func([]StreamEvent) error); ok { - r0 = returnFunc(events) - } else { - r0 = ret.Error(0) - } - return r0 +func (_mock *MockSubscriptionEventUpdater) Update(events []StreamEvent) { + _mock.Called(events) + return } // MockSubscriptionEventUpdater_Update_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Update' @@ -1250,12 +1239,12 @@ func (_c *MockSubscriptionEventUpdater_Update_Call) Run(run func(events []Stream return _c } -func (_c *MockSubscriptionEventUpdater_Update_Call) Return(err error) *MockSubscriptionEventUpdater_Update_Call { - _c.Call.Return(err) +func (_c *MockSubscriptionEventUpdater_Update_Call) Return() *MockSubscriptionEventUpdater_Update_Call { + _c.Call.Return() return _c } -func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(events []StreamEvent) error) *MockSubscriptionEventUpdater_Update_Call { - _c.Call.Return(run) +func (_c *MockSubscriptionEventUpdater_Update_Call) RunAndReturn(run func(events []StreamEvent)) *MockSubscriptionEventUpdater_Update_Call { + _c.Run(run) return _c } diff --git a/router/pkg/pubsub/datasource/mocks_resolve.go b/router/pkg/pubsub/datasource/mocks_resolve.go index 3efc24b405..19bad89c16 100644 --- a/router/pkg/pubsub/datasource/mocks_resolve.go +++ b/router/pkg/pubsub/datasource/mocks_resolve.go @@ -5,6 +5,8 @@ package datasource import ( + "context" + mock "github.com/stretchr/testify/mock" "github.com/wundergraph/graphql-go-tools/v2/pkg/engine/resolve" ) @@ -76,6 +78,52 @@ func (_c *MockSubscriptionUpdater_Close_Call) RunAndReturn(run func(kind resolve return _c } +// CloseSubscription provides a mock function for the type MockSubscriptionUpdater +func (_mock *MockSubscriptionUpdater) CloseSubscription(kind resolve.SubscriptionCloseKind, id resolve.SubscriptionIdentifier) { + _mock.Called(kind, id) + return +} + +// MockSubscriptionUpdater_CloseSubscription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CloseSubscription' +type MockSubscriptionUpdater_CloseSubscription_Call struct { + *mock.Call +} + +// CloseSubscription is a helper method to define mock.On call +// - kind resolve.SubscriptionCloseKind +// - id resolve.SubscriptionIdentifier +func (_e *MockSubscriptionUpdater_Expecter) CloseSubscription(kind interface{}, id interface{}) *MockSubscriptionUpdater_CloseSubscription_Call { + return &MockSubscriptionUpdater_CloseSubscription_Call{Call: _e.mock.On("CloseSubscription", kind, id)} +} + +func (_c *MockSubscriptionUpdater_CloseSubscription_Call) Run(run func(kind resolve.SubscriptionCloseKind, id resolve.SubscriptionIdentifier)) *MockSubscriptionUpdater_CloseSubscription_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 resolve.SubscriptionCloseKind + if args[0] != nil { + arg0 = args[0].(resolve.SubscriptionCloseKind) + } + var arg1 resolve.SubscriptionIdentifier + if args[1] != nil { + arg1 = args[1].(resolve.SubscriptionIdentifier) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockSubscriptionUpdater_CloseSubscription_Call) Return() *MockSubscriptionUpdater_CloseSubscription_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionUpdater_CloseSubscription_Call) RunAndReturn(run func(kind resolve.SubscriptionCloseKind, id resolve.SubscriptionIdentifier)) *MockSubscriptionUpdater_CloseSubscription_Call { + _c.Run(run) + return _c +} + // Complete provides a mock function for the type MockSubscriptionUpdater func (_mock *MockSubscriptionUpdater) Complete() { _mock.Called() @@ -109,6 +157,52 @@ func (_c *MockSubscriptionUpdater_Complete_Call) RunAndReturn(run func()) *MockS return _c } +// Subscriptions provides a mock function for the type MockSubscriptionUpdater +func (_mock *MockSubscriptionUpdater) Subscriptions() map[context.Context]resolve.SubscriptionIdentifier { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Subscriptions") + } + + var r0 map[context.Context]resolve.SubscriptionIdentifier + if returnFunc, ok := ret.Get(0).(func() map[context.Context]resolve.SubscriptionIdentifier); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[context.Context]resolve.SubscriptionIdentifier) + } + } + return r0 +} + +// MockSubscriptionUpdater_Subscriptions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscriptions' +type MockSubscriptionUpdater_Subscriptions_Call struct { + *mock.Call +} + +// Subscriptions is a helper method to define mock.On call +func (_e *MockSubscriptionUpdater_Expecter) Subscriptions() *MockSubscriptionUpdater_Subscriptions_Call { + return &MockSubscriptionUpdater_Subscriptions_Call{Call: _e.mock.On("Subscriptions")} +} + +func (_c *MockSubscriptionUpdater_Subscriptions_Call) Run(run func()) *MockSubscriptionUpdater_Subscriptions_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockSubscriptionUpdater_Subscriptions_Call) Return(contextToSubscriptionIdentifier map[context.Context]resolve.SubscriptionIdentifier) *MockSubscriptionUpdater_Subscriptions_Call { + _c.Call.Return(contextToSubscriptionIdentifier) + return _c +} + +func (_c *MockSubscriptionUpdater_Subscriptions_Call) RunAndReturn(run func() map[context.Context]resolve.SubscriptionIdentifier) *MockSubscriptionUpdater_Subscriptions_Call { + _c.Call.Return(run) + return _c +} + // Update provides a mock function for the type MockSubscriptionUpdater func (_mock *MockSubscriptionUpdater) Update(data []byte) { _mock.Called(data) @@ -148,3 +242,49 @@ func (_c *MockSubscriptionUpdater_Update_Call) RunAndReturn(run func(data []byte _c.Run(run) return _c } + +// UpdateSubscription provides a mock function for the type MockSubscriptionUpdater +func (_mock *MockSubscriptionUpdater) UpdateSubscription(id resolve.SubscriptionIdentifier, data []byte) { + _mock.Called(id, data) + return +} + +// MockSubscriptionUpdater_UpdateSubscription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateSubscription' +type MockSubscriptionUpdater_UpdateSubscription_Call struct { + *mock.Call +} + +// UpdateSubscription is a helper method to define mock.On call +// - id resolve.SubscriptionIdentifier +// - data []byte +func (_e *MockSubscriptionUpdater_Expecter) UpdateSubscription(id interface{}, data interface{}) *MockSubscriptionUpdater_UpdateSubscription_Call { + return &MockSubscriptionUpdater_UpdateSubscription_Call{Call: _e.mock.On("UpdateSubscription", id, data)} +} + +func (_c *MockSubscriptionUpdater_UpdateSubscription_Call) Run(run func(id resolve.SubscriptionIdentifier, data []byte)) *MockSubscriptionUpdater_UpdateSubscription_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 resolve.SubscriptionIdentifier + if args[0] != nil { + arg0 = args[0].(resolve.SubscriptionIdentifier) + } + var arg1 []byte + if args[1] != nil { + arg1 = args[1].([]byte) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockSubscriptionUpdater_UpdateSubscription_Call) Return() *MockSubscriptionUpdater_UpdateSubscription_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSubscriptionUpdater_UpdateSubscription_Call) RunAndReturn(run func(id resolve.SubscriptionIdentifier, data []byte)) *MockSubscriptionUpdater_UpdateSubscription_Call { + _c.Run(run) + return _c +} diff --git a/router/pkg/pubsub/datasource/provider.go b/router/pkg/pubsub/datasource/provider.go index b9f5903485..4577bfee32 100644 --- a/router/pkg/pubsub/datasource/provider.go +++ b/router/pkg/pubsub/datasource/provider.go @@ -58,6 +58,7 @@ const ( // there could be other common fields in the future, but for now we only have data type StreamEvent interface { GetData() []byte + Clone() StreamEvent } // SubscriptionEventConfiguration is the interface that all subscription event configurations must implement diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index 51c7b239f3..2ff1ed57da 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -19,6 +19,11 @@ func (e *testEvent) GetData() []byte { return e.data } +func (e *testEvent) Clone() StreamEvent { + e2 := *e + return &e2 +} + type testSubscriptionConfig struct { providerID string providerType ProviderType diff --git a/router/pkg/pubsub/datasource/subscription_datasource.go b/router/pkg/pubsub/datasource/subscription_datasource.go index 9590cd91a8..16ec03171a 100644 --- a/router/pkg/pubsub/datasource/subscription_datasource.go +++ b/router/pkg/pubsub/datasource/subscription_datasource.go @@ -41,7 +41,7 @@ func (s *PubSubSubscriptionDataSource[C]) Start(ctx *resolve.Context, input []by return errors.New("invalid subscription configuration") } - return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(ctx.Context(), conf, s.hooks, updater, s.logger)) + return s.pubSub.Subscribe(ctx.Context(), conf, NewSubscriptionEventUpdater(conf, s.hooks, updater, s.logger)) } func (s *PubSubSubscriptionDataSource[C]) SubscriptionOnStart(ctx resolve.StartupHookContext, input []byte) (err error) { diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 7fc0fe0725..57133ee56f 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -11,7 +11,7 @@ import ( // that provides a way to send the event struct instead of the raw data // It is used to give access to the event additional fields to the hooks. type SubscriptionEventUpdater interface { - Update(events []StreamEvent) error + Update(events []StreamEvent) Complete() Close(kind resolve.SubscriptionCloseKind) SetHooks(hooks Hooks) @@ -19,50 +19,50 @@ type SubscriptionEventUpdater interface { type subscriptionEventUpdater struct { eventUpdater resolve.SubscriptionUpdater - ctx context.Context subscriptionEventConfiguration SubscriptionEventConfiguration hooks Hooks logger *zap.Logger } -func (s *subscriptionEventUpdater) updateEvents(events []StreamEvent) { - for _, event := range events { - s.eventUpdater.Update(event.GetData()) - } -} - -func (s *subscriptionEventUpdater) Update(events []StreamEvent) error { +func (s *subscriptionEventUpdater) Update(events []StreamEvent) { if len(s.hooks.OnReceiveEvents) == 0 { - s.updateEvents(events) - return nil + for _, event := range events { + s.eventUpdater.Update(event.GetData()) + } + return } - - processedEvents, err := applyStreamEventHooks(s.ctx, s.subscriptionEventConfiguration, events, s.hooks.OnReceiveEvents) - // updates the events even if the hooks fail - // if a hook doesn't want to send the events, it should return no events! - s.updateEvents(processedEvents) - if err != nil { - // Check if the error is a StreamHookError and should close the subscription - // We use type assertion to check for the CloseSubscription method without importing core - if hookErr, ok := err.(ErrorWithCloseSubscription); ok { - if hookErr.CloseSubscription() { - // If CloseSubscription is true, return the error to close the subscription - return err - } + // If there are hooks, we should apply them separated for each subscription + for ctx, subId := range s.eventUpdater.Subscriptions() { + processedEvents, err := applyStreamEventHooks( + ctx, + s.subscriptionEventConfiguration, + events, + s.hooks.OnReceiveEvents, + ) + // updates the events even if the hooks fail + // if a hook doesn't want to send the events, it should return no events! + for _, event := range processedEvents { + s.eventUpdater.UpdateSubscription(subId, event.GetData()) } - // For all other errors, just log them and continue - if s.logger != nil { - s.logger.Error( - "An error occurred while processing stream events hooks", - zap.Error(err), - zap.String("provider_type", string(s.subscriptionEventConfiguration.ProviderType())), - zap.String("provider_id", s.subscriptionEventConfiguration.ProviderID()), - zap.String("field_name", s.subscriptionEventConfiguration.RootFieldName()), - ) + if err != nil { + // For all errors, log them + if s.logger != nil { + s.logger.Error( + "An error occurred while processing stream events hooks", + zap.Error(err), + zap.String("provider_type", string(s.subscriptionEventConfiguration.ProviderType())), + zap.String("provider_id", s.subscriptionEventConfiguration.ProviderID()), + zap.String("field_name", s.subscriptionEventConfiguration.RootFieldName()), + ) + } + // Check if the error is a StreamHookError and should close the subscription + // We use type assertion to check for the CloseSubscription method without importing core + if hookErr, ok := err.(ErrorWithCloseSubscription); ok && hookErr.CloseSubscription() { + s.eventUpdater.CloseSubscription(resolve.SubscriptionCloseKindNormal, subId) + return + } } } - - return nil } func (s *subscriptionEventUpdater) Complete() { @@ -84,7 +84,13 @@ func applyStreamEventHooks( cfg SubscriptionEventConfiguration, events []StreamEvent, hooks []OnReceiveEventsFn) ([]StreamEvent, error) { - currentEvents := events + // Copy the events to avoid modifying the original slice + currentEvents := make([]StreamEvent, len(events), len(events)) + for i, event := range events { + currentEvents[i] = event.Clone() + } + // Apply each hook in sequence, passing the result of one as the input to the next + // If any hook returns an error, stop processing and return the error for _, hook := range hooks { var err error currentEvents, err = hook(ctx, cfg, currentEvents) @@ -96,14 +102,12 @@ func applyStreamEventHooks( } func NewSubscriptionEventUpdater( - ctx context.Context, cfg SubscriptionEventConfiguration, hooks Hooks, eventUpdater resolve.SubscriptionUpdater, logger *zap.Logger, ) SubscriptionEventUpdater { return &subscriptionEventUpdater{ - ctx: ctx, subscriptionEventConfiguration: cfg, hooks: hooks, eventUpdater: eventUpdater, diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go index fc32cb33a0..388561328b 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater_test.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -38,7 +38,6 @@ type receivedHooksArgs struct { func TestSubscriptionEventUpdater_Update_NoHooks(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -55,19 +54,15 @@ func TestSubscriptionEventUpdater_Update_NoHooks(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, // No hooks } - err := updater.Update(events) - - assert.NoError(t, err) + updater.Update(events) } -func TestSubscriptionEventUpdater_Update_WithHooks_Success(t *testing.T) { +func TestSubscriptionEventUpdater_UpdateSubscription_WithHooks_Success(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -87,19 +82,22 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Success(t *testing.T) { return modifiedEvents, nil } - // Expect call to Update with modified data - mockUpdater.On("Update", []byte("modified data")).Return() + // Expect call to UpdateSubscription with modified data + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("UpdateSubscription", subId, []byte("modified data")).Return() + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } - err := updater.Update(originalEvents) + updater.Update(originalEvents) select { case receivedArgs := <-receivedArgs: @@ -108,13 +106,10 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Success(t *testing.T) { case <-time.After(1 * time.Second): t.Fatal("timeout waiting for events") } - - assert.NoError(t, err) } -func TestSubscriptionEventUpdater_Update_WithHooks_Error(t *testing.T) { +func TestSubscriptionEventUpdater_UpdateSubscriptions_WithHooks_Error(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -130,27 +125,30 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error(t *testing.T) { return nil, hookError } - // Should not call Update on eventUpdater since hook fails + // Expect call to UpdateSubscription with modified data + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) + + // Should not call Update or UpdateSubscription on eventUpdater since hook fails updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } - err := updater.Update(events) + updater.Update(events) - // With the new behavior, errors are logged and nil is returned - assert.NoError(t, err) - // Assert that Update was not called on the eventUpdater + // Assert that Update and UpdateSubscription were not called on the eventUpdater mockUpdater.AssertNotCalled(t, "Update") + mockUpdater.AssertNotCalled(t, "UpdateSubscription") } func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -173,19 +171,22 @@ func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) return []StreamEvent{&testEvent{data: []byte("modified by hook2")}}, nil } - // Expect call to Update with data modified by hook2 (last hook) - mockUpdater.On("Update", []byte("modified by hook2")).Return() + // Expect call to UpdateSubscription with modified data + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("UpdateSubscription", subId, []byte("modified by hook2")).Return() + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ OnReceiveEvents: []OnReceiveEventsFn{hook1, hook2}, }, } - err := updater.Update(originalEvents) + updater.Update(originalEvents) select { case receivedArgs1 := <-receivedArgs1: @@ -202,13 +203,10 @@ func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) case <-time.After(1 * time.Second): t.Fatal("timeout waiting for events") } - - assert.NoError(t, err) } func TestSubscriptionEventUpdater_Complete(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -219,7 +217,6 @@ func TestSubscriptionEventUpdater_Complete(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, } @@ -229,7 +226,6 @@ func TestSubscriptionEventUpdater_Complete(t *testing.T) { func TestSubscriptionEventUpdater_Close(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -241,7 +237,6 @@ func TestSubscriptionEventUpdater_Close(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, } @@ -251,7 +246,6 @@ func TestSubscriptionEventUpdater_Close(t *testing.T) { func TestSubscriptionEventUpdater_SetHooks(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -268,7 +262,6 @@ func TestSubscriptionEventUpdater_SetHooks(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, } @@ -280,7 +273,6 @@ func TestSubscriptionEventUpdater_SetHooks(t *testing.T) { func TestNewSubscriptionEventUpdater(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -295,7 +287,7 @@ func TestNewSubscriptionEventUpdater(t *testing.T) { OnReceiveEvents: []OnReceiveEventsFn{testHook}, } - updater := NewSubscriptionEventUpdater(ctx, config, hooks, mockUpdater, zap.NewNop()) + updater := NewSubscriptionEventUpdater(config, hooks, mockUpdater, zap.NewNop()) assert.NotNil(t, updater) @@ -303,7 +295,6 @@ func TestNewSubscriptionEventUpdater(t *testing.T) { var concreteUpdater *subscriptionEventUpdater assert.IsType(t, concreteUpdater, updater) concreteUpdater = updater.(*subscriptionEventUpdater) - assert.Equal(t, ctx, concreteUpdater.ctx) assert.Equal(t, config, concreteUpdater.subscriptionEventConfiguration) assert.Equal(t, hooks, concreteUpdater.hooks) assert.Equal(t, mockUpdater, concreteUpdater.eventUpdater) @@ -487,7 +478,6 @@ func TestApplyStreamEventHooks_MultipleHooks_MiddleHookError(t *testing.T) { // Test the updateEvents method indirectly through Update method func TestSubscriptionEventUpdater_UpdateEvents_EmptyEvents(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -497,14 +487,12 @@ func TestSubscriptionEventUpdater_UpdateEvents_EmptyEvents(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, // No hooks } - err := updater.Update(events) + updater.Update(events) - assert.NoError(t, err) // No calls to Update should be made for empty events mockUpdater.AssertNotCalled(t, "Update") } @@ -522,7 +510,6 @@ func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -533,7 +520,6 @@ func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{}, } @@ -543,9 +529,8 @@ func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { } } -func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t *testing.T) { +func TestSubscriptionEventUpdater_UpdateSubscription_WithStreamHookError_CloseSubscription(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -568,24 +553,25 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_CloseSubscription(t updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } - mockUpdater.On("Update", []byte("test data")).Return() - err := updater.Update(events) + // Expect call to UpdateSubscription with modified data + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("UpdateSubscription", subId, []byte("test data")).Return() + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) + mockUpdater.On("CloseSubscription", resolve.SubscriptionCloseKindNormal, subId).Return() - // Should return the error when CloseSubscription is true - assert.Error(t, err) - assert.Equal(t, mockHookError, err) + updater.Update(events) } -func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription(t *testing.T) { +func TestSubscriptionEventUpdater_UpdateSubscription_WithStreamHookError_NoCloseSubscription(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -608,25 +594,28 @@ func TestSubscriptionEventUpdater_Update_WithStreamHookError_NoCloseSubscription updater := &subscriptionEventUpdater{ eventUpdater: mockUpdater, - ctx: ctx, subscriptionEventConfiguration: config, hooks: Hooks{ OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, } - mockUpdater.On("Update", []byte("test data")).Return() - err := updater.Update(events) + // Expect call to UpdateSubscription with modified data + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("UpdateSubscription", subId, []byte("test data")).Return() + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) + + updater.Update(events) - // Should return nil when CloseSubscription is false (error is logged) - assert.NoError(t, err) // Assert that Update was not called on the eventUpdater mockUpdater.AssertNotCalled(t, "Update") + mockUpdater.AssertNotCalled(t, "CloseSubscription") } -func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *testing.T) { +func TestSubscriptionEventUpdater_UpdateSubscription_WithHooks_Error_LoggerWritesError(t *testing.T) { mockUpdater := NewMockSubscriptionUpdater(t) - ctx := context.Background() config := &testSubscriptionEventConfig{ providerID: "test-provider", providerType: ProviderTypeNats, @@ -647,16 +636,19 @@ func TestSubscriptionEventUpdater_Update_WithHooks_Error_LoggerWritesError(t *te // Test with a real zap logger to verify error logging behavior // The logger.Error() call should be executed when an error occurs - updater := NewSubscriptionEventUpdater(ctx, config, Hooks{ + updater := NewSubscriptionEventUpdater(config, Hooks{ OnReceiveEvents: []OnReceiveEventsFn{testHook}, }, mockUpdater, logger) - err := updater.Update(events) + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) + + updater.Update(events) - // Should return nil when error is logged - assert.NoError(t, err) // Assert that Update was not called on the eventUpdater - mockUpdater.AssertNotCalled(t, "Update") + mockUpdater.AssertNotCalled(t, "UpdateSubscription") msgs := logObserver.FilterMessageSnippet("An error occurred while processing stream events hooks").TakeAll() assert.Equal(t, 1, len(msgs)) diff --git a/router/pkg/pubsub/kafka/adapter.go b/router/pkg/pubsub/kafka/adapter.go index d4395651be..a9bdf81a5a 100644 --- a/router/pkg/pubsub/kafka/adapter.go +++ b/router/pkg/pubsub/kafka/adapter.go @@ -88,15 +88,11 @@ func (p *ProviderAdapter) topicPoller(ctx context.Context, client *kgo.Client, u headers[header.Key] = header.Value } - err := updater.Update([]datasource.StreamEvent{&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: r.Value, Headers: headers, Key: r.Key, }}) - // if an error occurred while updating the subscription, should exit the poller - if err != nil { - return err - } } } } diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 28cbcb6d8f..68a0181ff4 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -25,6 +25,11 @@ func (e *Event) GetData() []byte { return e.Data } +func (e *Event) Clone() datasource.StreamEvent { + e2 := *e + return &e2 +} + // SubscriptionEventConfiguration is a public type that is used to allow access to custom fields // of the provider type SubscriptionEventConfiguration struct { @@ -163,7 +168,7 @@ func (s *PublishDataSource) Load(ctx context.Context, input []byte, out *bytes.B if err := s.pubSub.Publish(ctx, publishData.PublishEventConfiguration(), []datasource.StreamEvent{&publishData.Event}); err != nil { // err will not be returned but only logged inside PubSubProvider.Publish to avoid a "unable to fetch from subgraph" error - _, errWrite := io.WriteString(out, `{"success": false}`) + _, errWrite := io.WriteString(out, `{"success": false}`) return errWrite } _, errWrite := io.WriteString(out, `{"success": true}`) diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index eb12728c9f..6695d83614 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -132,22 +132,10 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip for msg := range msgBatch.Messages() { log.Debug("subscription update", zap.String("message_subject", msg.Subject()), zap.ByteString("data", msg.Data())) - updateErr := updater.Update([]datasource.StreamEvent{&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data(), Headers: msg.Headers(), }}) - if updateErr != nil { - // If the update fails, we do not acknowledge the message - log.Error( - "error updating subscription, stopping subscription", - zap.Error(updateErr), - zap.String("message_subject", msg.Subject()), - zap.String("provider_id", subConf.ProviderID()), - zap.String("provider_type", string(subConf.ProviderType())), - zap.String("field_name", subConf.RootFieldName()), - ) - return - } // Acknowledge the message after it has been processed ackErr := msg.Ack() @@ -184,29 +172,10 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, cfg datasource.Subscrip select { case msg := <-msgChan: log.Debug("subscription update", zap.String("message_subject", msg.Subject), zap.ByteString("data", msg.Data)) - updateErr := updater.Update([]datasource.StreamEvent{&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: msg.Data, Headers: msg.Header, }}) - if updateErr != nil { - // If the update fails, we log the error and unsubscribe from all subscriptions - log.Error( - "error updating subscription, stopping subscription", - zap.Error(updateErr), - zap.String("message_subject", msg.Subject), - zap.String("provider_id", subConf.ProviderID()), - zap.String("provider_type", string(subConf.ProviderType())), - zap.String("field_name", subConf.RootFieldName()), - ) - for _, subscription := range subscriptions { - if err := subscription.Unsubscribe(); err != nil { - log.Error("unsubscribing from NATS subject after an error on updating subscription", - zap.Error(err), zap.String("subject", subscription.Subject), - ) - } - } - return - } case <-p.ctx.Done(): // When the application context is done, we stop the subscriptions for _, subscription := range subscriptions { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index 8c5b34af71..b430128970 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -24,6 +24,11 @@ func (e *Event) GetData() []byte { return e.Data } +func (e *Event) Clone() datasource.StreamEvent { + e2 := *e + return &e2 +} + type StreamConfiguration struct { Consumer string `json:"consumer"` ConsumerInactiveThreshold int32 `json:"consumerInactiveThreshold"` diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index f05736351a..0c844b4d93 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -114,22 +114,9 @@ func (p *ProviderAdapter) Subscribe(ctx context.Context, conf datasource.Subscri return } log.Debug("subscription update", zap.String("message_channel", msg.Channel), zap.String("data", msg.Payload)) - updateErr := updater.Update([]datasource.StreamEvent{&Event{ + updater.Update([]datasource.StreamEvent{&Event{ Data: []byte(msg.Payload), }}) - if updateErr != nil { - log.Error( - "error updating subscription, stopping subscription", - zap.Error(updateErr), - zap.String("message_channel", msg.Channel), - zap.String("provider_id", conf.ProviderID()), - zap.String("provider_type", string(conf.ProviderType())), - zap.String("field_name", conf.RootFieldName()), - ) - // If the error is not recoverable, we stop the subscription - cleanup() - return - } case <-p.ctx.Done(): // When the application context is done, we stop the subscription if it is not already done log.Debug("application context done, stopping subscription") diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index b59b9ab0b5..dec3ee860f 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -23,6 +23,11 @@ func (e *Event) GetData() []byte { return e.Data } +func (e *Event) Clone() datasource.StreamEvent { + e2 := *e + return &e2 +} + // SubscriptionEventConfiguration contains configuration for subscription events type SubscriptionEventConfiguration struct { Provider string `json:"providerId"` From a235b91ba840cb6972dd6b5a31d42e567e8df6ef Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 12:04:35 +0200 Subject: [PATCH 162/173] chore: update graphql-go-tools dependency to v2.0.0-rc.213.0.20250924095800-f87d764344f3 --- router-tests/go.mod | 2 +- router-tests/go.sum | 4 ++-- router/go.mod | 2 +- router/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/router-tests/go.mod b/router-tests/go.mod index 337cfcc049..ad605fe407 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/wundergraph/cosmo/demo v0.0.0-20250718181713-66224598e91f github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250909162318-90dba997c711 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 4a1ba9c1f1..381a2a85be 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -325,8 +325,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e h1:IJ2VFXTW2m6cmethfzwwEfqwf1voHs0QEzdphwY7ViM= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3 h1:kaXHYaLwohwTXrbyhiIqyOHK1Rp6EYcl1JL7lySpD5Q= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/go.mod b/router/go.mod index 7ac0e475b7..e9aa192b4b 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 diff --git a/router/go.sum b/router/go.sum index 927d7c3d1c..3bd5ba4d11 100644 --- a/router/go.sum +++ b/router/go.sum @@ -290,8 +290,8 @@ github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e h1:IJ2VFXTW2m6cmethfzwwEfqwf1voHs0QEzdphwY7ViM= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250909161213-a0024f08f31e/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3 h1:kaXHYaLwohwTXrbyhiIqyOHK1Rp6EYcl1JL7lySpD5Q= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.213.0.20250924095800-f87d764344f3/go.mod h1:DnYY1alnsgzkanSwbFiFIdXKOuf8dHQWQ2P4BzTc6aI= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= From b19956a43e2023b8ae620d644b8fd1781bf45e6e Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 12:12:25 +0200 Subject: [PATCH 163/173] chore: fix S1019 --- router/pkg/pubsub/datasource/subscription_event_updater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 57133ee56f..ae140f26a0 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -85,7 +85,7 @@ func applyStreamEventHooks( events []StreamEvent, hooks []OnReceiveEventsFn) ([]StreamEvent, error) { // Copy the events to avoid modifying the original slice - currentEvents := make([]StreamEvent, len(events), len(events)) + currentEvents := make([]StreamEvent, len(events)) for i, event := range events { currentEvents[i] = event.Clone() } From 14e3c36a8ec0d36d2b2a0691371e93aa184a98b3 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 12:24:35 +0200 Subject: [PATCH 164/173] fix: test should use the new module id streamReceiveModule --- router-tests/modules/stream_receive_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index de2d24d6ae..61bfe28ace 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -364,7 +364,7 @@ func TestReceiveHook(t *testing.T) { cfg := config.Config{ Graph: config.Graph{}, Modules: map[string]interface{}{ - "streamBatchModule": stream_receive.StreamReceiveModule{ + "streamReceiveModule": stream_receive.StreamReceiveModule{ Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { return nil, &errorWithCloseSubscription{err: errors.New("test error from streamevents hook")} }, From 9a7e08f012b8872a654bd2aec0e1b18e12b5f548 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 12:29:06 +0200 Subject: [PATCH 165/173] fix: remove unnecessary Kafka connection log verification and streamline subscription closure handling --- router-tests/modules/stream_receive_test.go | 5 +---- router/pkg/pubsub/datasource/subscription_event_updater.go | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 61bfe28ace..857960a73c 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -428,10 +428,7 @@ func TestReceiveHook(t *testing.T) { require.NoError(t, err) }, "client should have completed when server closed connection") - // Verify that Kafka connections are also closed by checking for "poller canceled" log messages - // The Kafka adapter logs this when connections are closed due to context cancellation - kafkaPollerLogs := xEnv.Observer().FilterMessage("poller error") - assert.GreaterOrEqual(t, len(kafkaPollerLogs.All()), 1, "Expected at least one Kafka poller to be canceled") + xEnv.WaitForTriggerCount(0, Timeout) }) }) } diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index ae140f26a0..82ca60007e 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -59,7 +59,6 @@ func (s *subscriptionEventUpdater) Update(events []StreamEvent) { // We use type assertion to check for the CloseSubscription method without importing core if hookErr, ok := err.(ErrorWithCloseSubscription); ok && hookErr.CloseSubscription() { s.eventUpdater.CloseSubscription(resolve.SubscriptionCloseKindNormal, subId) - return } } } From d61dcde7fea216285fc78208313c7d90dbfed94c Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 24 Sep 2025 16:54:20 +0200 Subject: [PATCH 166/173] chore: add a test to verify that StreamReceiveEventHook can access headers --- router-tests/modules/stream_receive_test.go | 99 +++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 857960a73c..b5106a0a03 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -358,6 +358,105 @@ func TestReceiveHook(t *testing.T) { }) }) + t.Run("Test Receive hook can access custom header", func(t *testing.T) { + t.Parallel() + + customHeader := http.CanonicalHeaderKey("X-Custom-Header") + + cfg := config.Config{ + Graph: config.Graph{}, + + Modules: map[string]interface{}{ + "streamReceiveModule": stream_receive.StreamReceiveModule{ + Callback: func(ctx core.StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { + if val, ok := ctx.Request().Header[customHeader]; !ok || val[0] != "Test" { + return events, nil + } + for _, event := range events { + evt, ok := event.(*kafka.Event) + if !ok { + continue + } + evt.Data = []byte(`{"__typename":"Employee","id": 3,"update":{"name":"foo"}}`) + } + + return events, nil + }, + }, + }, + } + + testenv.Run(t, &testenv.Config{ + RouterConfigJSONTemplate: testenv.ConfigWithEdfsKafkaJSONTemplate, + EnableKafka: true, + RouterOptions: []core.Option{ + core.WithModulesConfig(cfg.Modules), + core.WithCustomModules(&stream_receive.StreamReceiveModule{}), + }, + LogObservation: testenv.LogObservationConfig{ + Enabled: true, + LogLevel: zapcore.InfoLevel, + }, + }, func(t *testing.T, xEnv *testenv.Environment) { + topics := []string{"employeeUpdated"} + events.KafkaEnsureTopicExists(t, xEnv, time.Second, topics...) + + var subscriptionOne struct { + employeeUpdatedMyKafka struct { + ID float64 `graphql:"id"` + Details struct { + Forename string `graphql:"forename"` + Surname string `graphql:"surname"` + } `graphql:"details"` + } `graphql:"employeeUpdatedMyKafka(employeeID: 3)"` + } + headers := http.Header{ + customHeader: []string{"Test"}, + } + + surl := xEnv.GraphQLWebSocketSubscriptionURL() + client := graphql.NewSubscriptionClient(surl) + client.WithWebSocketOptions(graphql.WebsocketOptions{ + HTTPHeader: headers, + }) + + subscriptionArgsCh := make(chan kafkaSubscriptionArgs) + subscriptionOneID, err := client.Subscribe(&subscriptionOne, nil, func(dataValue []byte, errValue error) error { + subscriptionArgsCh <- kafkaSubscriptionArgs{ + dataValue: dataValue, + errValue: errValue, + } + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, subscriptionOneID) + + clientRunCh := make(chan error) + go func() { + clientRunCh <- client.Run() + }() + + xEnv.WaitForSubscriptionCount(1, Timeout) + + events.ProduceKafkaMessage(t, xEnv, Timeout, topics[0], `{"__typename":"Employee","id": 1,"update":{"name":"foo"}}`) + + testenv.AwaitChannelWithT(t, Timeout, subscriptionArgsCh, func(t *testing.T, args kafkaSubscriptionArgs) { + require.NoError(t, args.errValue) + assert.JSONEq(t, `{"employeeUpdatedMyKafka":{"id":3,"details":{"forename":"Stefan","surname":"Avram"}}}`, string(args.dataValue)) + }) + + unSub1Err := client.Unsubscribe(subscriptionOneID) + require.NoError(t, unSub1Err) + require.NoError(t, client.Close()) + testenv.AwaitChannelWithT(t, Timeout, clientRunCh, func(t *testing.T, err error) { + require.NoError(t, err) + }, "unable to close client before timeout") + + requestLog := xEnv.Observer().FilterMessage("Stream Hook has been run") + assert.Len(t, requestLog.All(), 1) + }) + }) + t.Run("Test Batch hook error should close Kafka clients and subscriptions", func(t *testing.T) { t.Parallel() From 31a68cfcf226d872a232d4e5d6cfaf9094119791 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 25 Sep 2025 10:36:19 +0200 Subject: [PATCH 167/173] feat: enhance StreamReceiveEventHook documentation --- adr/cosmo-streams-v1.md | 38 ++++++++++++++-------------- router/core/subscriptions_modules.go | 10 +++++--- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index ccd7fa2ae1..92b1904add 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -21,7 +21,7 @@ The following interfaces will extend the existing logic in the custom modules. These provide additional control over subscriptions by providing hooks, which are invoked during specific events. - `SubscriptionOnStartHandler`: Called once at subscription start. -- `StreamReceiveEventHook`: Called each time a batch of events is received from the provider. +- `StreamReceiveEventHook`: Triggered for each client/subscription when a batch of events is received from the provider, prior to delivery. - `StreamPublishEventHook`: Called each time a batch of events is going to be sent to the provider. ```go @@ -104,7 +104,9 @@ type StreamReceiveEventHookContext interface { } type StreamReceiveEventHook interface { - // OnStreamEvents is called each time a batch of events is received from the provider + // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the client + // So for a single batch of events received from the provider, this hook will be called one time for each active subscription. + // It is important to optimize the logic inside this hook to avoid performance issues. // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. OnReceiveEvents(ctx StreamReceiveEventHookContext, events []StreamEvent) ([]StreamEvent, error) } @@ -167,7 +169,7 @@ type Employee @key(fields: "id", resolvable: false) { id: Int! @external } ``` -After publishing the schema, the developer will need to add the module to the cosmo streams engine. +After publishing the schema, the developer will need to add the module to the cosmo router. ### 2. Write the custom module @@ -201,28 +203,27 @@ func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHookContext, events []c return events, nil } - // check if the subject is the one expected by the module - natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) - if natsConfig.Subjects[0] != "employeeUpdates" { - return events, nil - } + // check if the subscription is the one expected by the module + if ctx.SubscriptionEventConfiguration().RootFieldName() != "employeeUpdates" { + return events, nil + } + + newEvents := make([]core.StreamEvent, 0, len(events)) // check if the client is authenticated if ctx.Authentication() == nil { // if the client is not authenticated, return no events - return events, nil + return newEvents, nil } // check if the client is allowed to subscribe to the stream clientAllowedEntitiesIds, found := ctx.Authentication().Claims()["allowedEntitiesIds"] if !found { - return events, fmt.Errorf("client is not allowed to subscribe to the stream") + return newEvents, fmt.Errorf("client is not allowed to subscribe to the stream") } - - newEvents := make([]core.StreamEvent, 0, len(events)) - + for _, evt := range events { - natsEvent, ok := evt.(*nats.NatsEvent); + natsEvent, ok := evt.(*nats.NatsEvent) if !ok { newEvents = append(newEvents, evt) continue @@ -345,11 +346,10 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error return nil } - // check if the subject is the one expected by the module - natsConfig := ctx.SubscriptionEventConfiguration().(*nats.SubscriptionEventConfiguration) - if natsConfig.Subjects[0] != "employeeUpdates" { - return nil - } + // check if the subscription is the one expected by the module + if ctx.SubscriptionEventConfiguration().RootFieldName() != "employeeUpdates" { + return nil + } // check if the client is authenticated if ctx.Authentication() == nil { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index d23bc54b05..6baaae6b9c 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -235,8 +235,11 @@ type StreamReceiveEventHookContext interface { } type StreamReceiveEventHook interface { - // OnReceiveEvents is called each time a batch of events is received from the provider - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the + // client. So for a single batch of events received from the provider, this hook will be called as many times as the + // number of active subscriptions. It is important to optimize the logic inside this hook to avoid performance issues. + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a + // StreamHookError. OnReceiveEvents(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } @@ -255,7 +258,8 @@ type StreamPublishEventHookContext interface { type StreamPublishEventHook interface { // OnPublishEvents is called each time a batch of events is going to be sent to the provider - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a + // StreamHookError. OnPublishEvents(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } From 3a0653cc884c07eec294caf200e2fdc3426ac8aa Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Tue, 30 Sep 2025 09:40:48 +0200 Subject: [PATCH 168/173] chore: fix events Clone method and remove useless redis Adapter abstraction --- adr/cosmo-streams-v1.md | 12 ++++++------ router/core/subscriptions_modules.go | 7 ++++--- router/pkg/pubsub/datasource/datasource.go | 2 +- router/pkg/pubsub/kafka/engine_datasource.go | 6 ++++++ router/pkg/pubsub/nats/adapter.go | 4 ---- router/pkg/pubsub/nats/engine_datasource.go | 6 ++++++ router/pkg/pubsub/redis/adapter.go | 14 +------------- router/pkg/pubsub/redis/engine_datasource.go | 4 ++-- .../pkg/pubsub/redis/engine_datasource_factory.go | 2 +- 9 files changed, 27 insertions(+), 30 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 92b1904add..5e7cb0b8d4 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -91,7 +91,7 @@ type SubscriptionOnStartHandler interface { } type StreamReceiveEventHookContext interface { - // Request is the original request received by the router. + // Request is the initial client request that started the subscription Request() *http.Request // Logger is the logger for the request Logger() *zap.Logger @@ -105,8 +105,8 @@ type StreamReceiveEventHookContext interface { type StreamReceiveEventHook interface { // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the client - // So for a single batch of events received from the provider, this hook will be called one time for each active subscription. - // It is important to optimize the logic inside this hook to avoid performance issues. + // So for a single batch of events received from the provider, this hook will be called one time for each active subscription. + // It is important to optimize the logic inside this hook to avoid performance issues. // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. OnReceiveEvents(ctx StreamReceiveEventHookContext, events []StreamEvent) ([]StreamEvent, error) } @@ -207,7 +207,7 @@ func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHookContext, events []c if ctx.SubscriptionEventConfiguration().RootFieldName() != "employeeUpdates" { return events, nil } - + newEvents := make([]core.StreamEvent, 0, len(events)) // check if the client is authenticated @@ -221,7 +221,7 @@ func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHookContext, events []c if !found { return newEvents, fmt.Errorf("client is not allowed to subscribe to the stream") } - + for _, evt := range events { natsEvent, ok := evt.(*nats.NatsEvent) if !ok { @@ -418,4 +418,4 @@ We could also generate the AsyncAPI specification from the schema and the events ## Generate hooks from AsyncAPI specifications -Building on the AsyncAPI integration, we could allow the user to define their streams using AsyncAPI and generate fully typesafe hooks with all events structures generated from the AsyncAPI specification. \ No newline at end of file +Building on the AsyncAPI integration, we could allow the user to define their streams using AsyncAPI and generate fully typesafe hooks with all events structures generated from the AsyncAPI specification. diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 6baaae6b9c..9881420ac5 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -222,7 +222,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext } type StreamReceiveEventHookContext interface { - // Request is the original request received by the router. + // Request is the initial client request that started the subscription Request() *http.Request // Logger is the logger for the request Logger() *zap.Logger @@ -236,8 +236,9 @@ type StreamReceiveEventHookContext interface { type StreamReceiveEventHook interface { // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the - // client. So for a single batch of events received from the provider, this hook will be called as many times as the - // number of active subscriptions. It is important to optimize the logic inside this hook to avoid performance issues. + // client. So for a single batch of events received from the provider, this hook will be called one time for each + // active subscription. + // It is important to optimize the logic inside this hook to avoid performance issues. // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a // StreamHookError. OnReceiveEvents(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) diff --git a/router/pkg/pubsub/datasource/datasource.go b/router/pkg/pubsub/datasource/datasource.go index 3cc3732612..b186041388 100644 --- a/router/pkg/pubsub/datasource/datasource.go +++ b/router/pkg/pubsub/datasource/datasource.go @@ -9,7 +9,7 @@ type SubscriptionDataSource interface { SubscriptionEventConfiguration(input []byte) (SubscriptionEventConfiguration, error) Start(ctx *resolve.Context, input []byte, updater resolve.SubscriptionUpdater) error UniqueRequestID(ctx *resolve.Context, input []byte, xxh *xxhash.Digest) (err error) - SetHooks(hoks Hooks) + SetHooks(hooks Hooks) } // EngineDataSourceFactory is the interface that all pubsub data sources must implement. diff --git a/router/pkg/pubsub/kafka/engine_datasource.go b/router/pkg/pubsub/kafka/engine_datasource.go index 68a0181ff4..00a38023ea 100644 --- a/router/pkg/pubsub/kafka/engine_datasource.go +++ b/router/pkg/pubsub/kafka/engine_datasource.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "slices" "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" @@ -27,6 +28,11 @@ func (e *Event) GetData() []byte { func (e *Event) Clone() datasource.StreamEvent { e2 := *e + e2.Data = slices.Clone(e.Data) + e2.Headers = make(map[string][]byte, len(e.Headers)) + for k, v := range e.Headers { + e2.Headers[k] = slices.Clone(v) + } return &e2 } diff --git a/router/pkg/pubsub/nats/adapter.go b/router/pkg/pubsub/nats/adapter.go index 6695d83614..48b5d8c1a7 100644 --- a/router/pkg/pubsub/nats/adapter.go +++ b/router/pkg/pubsub/nats/adapter.go @@ -219,10 +219,6 @@ func (p *ProviderAdapter) Publish(_ context.Context, conf datasource.PublishEven return datasource.NewError("nats client not initialized", nil) } - if len(events) == 0 { - return nil - } - log.Debug("publish", zap.Int("event_count", len(events))) for _, streamEvent := range events { diff --git a/router/pkg/pubsub/nats/engine_datasource.go b/router/pkg/pubsub/nats/engine_datasource.go index b430128970..3b2014a71a 100644 --- a/router/pkg/pubsub/nats/engine_datasource.go +++ b/router/pkg/pubsub/nats/engine_datasource.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "slices" "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" @@ -26,6 +27,11 @@ func (e *Event) GetData() []byte { func (e *Event) Clone() datasource.StreamEvent { e2 := *e + e2.Data = slices.Clone(e.Data) + e2.Headers = make(map[string][]string, len(e.Headers)) + for k, v := range e.Headers { + e2.Headers[k] = slices.Clone(v) + } return &e2 } diff --git a/router/pkg/pubsub/redis/adapter.go b/router/pkg/pubsub/redis/adapter.go index 0c844b4d93..b050f646e6 100644 --- a/router/pkg/pubsub/redis/adapter.go +++ b/router/pkg/pubsub/redis/adapter.go @@ -10,22 +10,10 @@ import ( "go.uber.org/zap" ) -// Adapter defines the methods that a Redis adapter should implement -type Adapter interface { - // Subscribe subscribes to the given events and sends updates to the updater - Subscribe(ctx context.Context, conf datasource.SubscriptionEventConfiguration, updater datasource.SubscriptionEventUpdater) error - // Publish publishes the given events to the specified channel - Publish(ctx context.Context, conf datasource.PublishEventConfiguration, events []datasource.StreamEvent) error - // Startup initializes the adapter - Startup(ctx context.Context) error - // Shutdown gracefully shuts down the adapter - Shutdown(ctx context.Context) error -} - // Ensure ProviderAdapter implements ProviderSubscriptionHooks var _ datasource.Adapter = (*ProviderAdapter)(nil) -func NewProviderAdapter(ctx context.Context, logger *zap.Logger, urls []string, clusterEnabled bool) Adapter { +func NewProviderAdapter(ctx context.Context, logger *zap.Logger, urls []string, clusterEnabled bool) datasource.Adapter { ctx, cancel := context.WithCancel(ctx) return &ProviderAdapter{ ctx: ctx, diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index dec3ee860f..f57e9501cd 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -95,7 +95,7 @@ func (p *PublishEventConfiguration) RootFieldName() string { // SubscriptionDataSource implements resolve.SubscriptionDataSource for Redis type SubscriptionDataSource struct { - pubSub Adapter + pubSub datasource.Adapter } func (s *SubscriptionDataSource) SubscriptionEventConfiguration(input []byte) datasource.SubscriptionEventConfiguration { @@ -150,7 +150,7 @@ func (s *SubscriptionDataSource) LoadInitialData(ctx context.Context) (initial [ // PublishDataSource implements resolve.DataSource for Redis publishing type PublishDataSource struct { - pubSub Adapter + pubSub datasource.Adapter } // Load processes a request to publish to Redis diff --git a/router/pkg/pubsub/redis/engine_datasource_factory.go b/router/pkg/pubsub/redis/engine_datasource_factory.go index 0d25716f3e..46f22e29b9 100644 --- a/router/pkg/pubsub/redis/engine_datasource_factory.go +++ b/router/pkg/pubsub/redis/engine_datasource_factory.go @@ -21,7 +21,7 @@ const ( // EngineDataSourceFactory implements the datasource.EngineDataSourceFactory interface for Redis type EngineDataSourceFactory struct { - RedisAdapter Adapter + RedisAdapter datasource.Adapter fieldName string eventType EventType From 9b0e12d48410ac42d857c288bcfd518ff485e08d Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Tue, 30 Sep 2025 13:40:23 +0200 Subject: [PATCH 169/173] chore: rename hooks to handlers --- adr/cosmo-streams-v1.md | 40 ++++++++++++++-------------- router/core/errors.go | 4 +-- router/core/graphql_handler.go | 2 +- router/core/router.go | 4 +-- router/core/router_config.go | 6 ++--- router/core/subscriptions_modules.go | 40 ++++++++++++++-------------- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 5e7cb0b8d4..01ae115c25 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -21,20 +21,20 @@ The following interfaces will extend the existing logic in the custom modules. These provide additional control over subscriptions by providing hooks, which are invoked during specific events. - `SubscriptionOnStartHandler`: Called once at subscription start. -- `StreamReceiveEventHook`: Triggered for each client/subscription when a batch of events is received from the provider, prior to delivery. -- `StreamPublishEventHook`: Called each time a batch of events is going to be sent to the provider. +- `StreamReceiveEventHandler`: Triggered for each client/subscription when a batch of events is received from the provider, prior to delivery. +- `StreamPublishEventHandler`: Called each time a batch of events is going to be sent to the provider. ```go // STRUCTURES TO BE ADDED TO PUBSUB PACKAGE type ProviderType string const ( - ProviderTypeNats ProviderType = "nats" + ProviderTypeNats ProviderType = "nats" ProviderTypeKafka ProviderType = "kafka" ProviderTypeRedis ProviderType = "redis" } -// StreamHookError is used to customize the error messages and the behavior -type StreamHookError struct { +// StreamHandlerError is used to customize the error messages and the behavior +type StreamHandlerError struct { HttpError core.HttpError CloseSubscription bool } @@ -68,7 +68,7 @@ type PublishEventConfiguration interface { RootFieldName() string } -type SubscriptionOnStartHookContext interface { +type SubscriptionOnStartHandlerContext interface { // Request is the original request received by the router. Request() *http.Request // Logger is the logger for the request @@ -86,11 +86,11 @@ type SubscriptionOnStartHookContext interface { type SubscriptionOnStartHandler interface { // OnSubscriptionOnStart is called once at subscription start - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHandlerError. + SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) error } -type StreamReceiveEventHookContext interface { +type StreamReceiveEventHandlerContext interface { // Request is the initial client request that started the subscription Request() *http.Request // Logger is the logger for the request @@ -103,15 +103,15 @@ type StreamReceiveEventHookContext interface { SubscriptionEventConfiguration() SubscriptionEventConfiguration } -type StreamReceiveEventHook interface { +type StreamReceiveEventHandler interface { // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the client // So for a single batch of events received from the provider, this hook will be called one time for each active subscription. // It is important to optimize the logic inside this hook to avoid performance issues. - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHookError. - OnReceiveEvents(ctx StreamReceiveEventHookContext, events []StreamEvent) ([]StreamEvent, error) + // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHandlerError. + OnReceiveEvents(ctx StreamReceiveEventHandlerContext, events []StreamEvent) ([]StreamEvent, error) } -type StreamPublishEventHookContext interface { +type StreamPublishEventHandlerContext interface { // Request is the original request received by the router. Request() *http.Request // Logger is the logger for the request @@ -124,10 +124,10 @@ type StreamPublishEventHookContext interface { PublishEventConfiguration() PublishEventConfiguration } -type StreamPublishEventHook interface { +type StreamPublishEventHandler interface { // OnPublishEvents is called each time a batch of events is going to be sent to the provider // Returning an error will result in an error being returned and the client will see the mutation failing - OnPublishEvents(ctx StreamPublishEventHookContext, events []StreamEvent) ([]StreamEvent, error) + OnPublishEvents(ctx StreamPublishEventHandlerContext, events []StreamEvent) ([]StreamEvent, error) } ``` @@ -192,7 +192,7 @@ func init() { type MyModule struct {} -func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHookContext, events []core.StreamEvent) ([]core.StreamEvent, error) { +func (m *MyModule) OnReceiveEvents(ctx StreamReceiveEventHandlerContext, events []core.StreamEvent) ([]core.StreamEvent, error) { // check if the provider is nats if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return events, nil @@ -280,7 +280,7 @@ func (m *MyModule) Module() core.ModuleInfo { // Interface guards var ( - _ core.StreamReceiveEventHook = (*MyModule)(nil) + _ core.StreamReceiveEventHandler = (*MyModule)(nil) ) ``` @@ -335,7 +335,7 @@ func init() { type MyModule struct {} -func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error { +func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) error { // check if the provider is nats if ctx.SubscriptionEventConfiguration().ProviderType() != pubsub.ProviderTypeNats { return nil @@ -354,7 +354,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error // check if the client is authenticated if ctx.Authentication() == nil { // if the client is not authenticated, return an error - return &StreamHookError{ + return &StreamHandlerError{ HttpError: core.HttpError{ Code: http.StatusUnauthorized, Message: "client is not authenticated", @@ -366,7 +366,7 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error // check if the client is allowed to subscribe to the stream clientAllowedEntitiesIds, found := ctx.Authentication().Claims()["readEmployee"] if !found { - return &StreamHookError{ + return &StreamHandlerError{ HttpError: core.HttpError{ Code: http.StatusForbidden, Message: "client is not allowed to read employees", diff --git a/router/core/errors.go b/router/core/errors.go index 44e05f327b..07f3175f64 100644 --- a/router/core/errors.go +++ b/router/core/errors.go @@ -90,8 +90,8 @@ func getErrorType(err error) errorType { if errors.As(err, &mergeResultErr) { return errorTypeMergeResult } - var streamHookErr *StreamHookError - if errors.As(err, &streamHookErr) { + var streamHandlerErr *StreamHandlerError + if errors.As(err, &streamHandlerErr) { return errorTypeStreamHookError } return errorTypeUnknown diff --git a/router/core/graphql_handler.go b/router/core/graphql_handler.go index f387d73e6c..58a9ba245c 100644 --- a/router/core/graphql_handler.go +++ b/router/core/graphql_handler.go @@ -401,7 +401,7 @@ func (h *GraphQLHandler) WriteError(ctx *resolve.Context, err error, res *resolv httpWriter.WriteHeader(http.StatusInternalServerError) } case errorTypeStreamHookError: - var streamHookErr *StreamHookError + var streamHookErr *StreamHandlerError if !errors.As(err, &streamHookErr) { response.Errors[0].Message = "Internal server error" return diff --git a/router/core/router.go b/router/core/router.go index 8955d14485..c2f8d05c28 100644 --- a/router/core/router.go +++ b/router/core/router.go @@ -646,11 +646,11 @@ func (r *Router) initModules(ctx context.Context) error { r.subscriptionHooks.onStart = append(r.subscriptionHooks.onStart, handler.SubscriptionOnStart) } - if handler, ok := moduleInstance.(StreamPublishEventHook); ok { + if handler, ok := moduleInstance.(StreamPublishEventHandler); ok { r.subscriptionHooks.onPublishEvents = append(r.subscriptionHooks.onPublishEvents, handler.OnPublishEvents) } - if handler, ok := moduleInstance.(StreamReceiveEventHook); ok { + if handler, ok := moduleInstance.(StreamReceiveEventHandler); ok { r.subscriptionHooks.onReceiveEvents = append(r.subscriptionHooks.onReceiveEvents, handler.OnReceiveEvents) } diff --git a/router/core/router_config.go b/router/core/router_config.go index 78d08c0c8a..79cf8e5658 100644 --- a/router/core/router_config.go +++ b/router/core/router_config.go @@ -27,9 +27,9 @@ import ( ) type subscriptionHooks struct { - onStart []func(ctx SubscriptionOnStartHookContext) error - onPublishEvents []func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) - onReceiveEvents []func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + onStart []func(ctx SubscriptionOnStartHandlerContext) error + onPublishEvents []func(ctx StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + onReceiveEvents []func(ctx StreamReceiveEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } type Config struct { diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 9881420ac5..28f1d59509 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -11,35 +11,35 @@ import ( "go.uber.org/zap" ) -// StreamHookError is used to customize the error messages and the behavior -type StreamHookError struct { +// StreamHandlerError is used to customize the error messages and the behavior +type StreamHandlerError struct { err error message string statusCode int code string } -func (e *StreamHookError) Error() string { +func (e *StreamHandlerError) Error() string { if e.err != nil { return e.err.Error() } return e.message } -func (e *StreamHookError) Message() string { +func (e *StreamHandlerError) Message() string { return e.message } -func (e *StreamHookError) StatusCode() int { +func (e *StreamHandlerError) StatusCode() int { return e.statusCode } -func (e *StreamHookError) Code() string { +func (e *StreamHandlerError) Code() string { return e.code } -func NewStreamHookError(err error, message string, statusCode int, code string) *StreamHookError { - return &StreamHookError{ +func NewStreamHookError(err error, message string, statusCode int, code string) *StreamHandlerError { + return &StreamHandlerError{ err: err, message: message, statusCode: statusCode, @@ -47,7 +47,7 @@ func NewStreamHookError(err error, message string, statusCode int, code string) } } -type SubscriptionOnStartHookContext interface { +type SubscriptionOnStartHandlerContext interface { // Request is the original request received by the router. Request() *http.Request // Logger is the logger for the request @@ -177,11 +177,11 @@ func (c *engineSubscriptionOnStartHookContext) SubscriptionEventConfiguration() type SubscriptionOnStartHandler interface { // SubscriptionOnStart is called once at subscription start // The error is propagated to the client. - SubscriptionOnStart(ctx SubscriptionOnStartHookContext) error + SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) error } // NewPubSubSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a pubsub.SubscriptionOnStartFn -func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) error) datasource.SubscriptionOnStartFn { +func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHandlerContext) error) datasource.SubscriptionOnStartFn { if fn == nil { return nil } @@ -202,7 +202,7 @@ func NewPubSubSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext } // NewEngineSubscriptionOnStartHook converts a SubscriptionOnStartHandler to a graphql_datasource.SubscriptionOnStartFn -func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext) error) graphql_datasource.SubscriptionOnStartFn { +func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHandlerContext) error) graphql_datasource.SubscriptionOnStartFn { if fn == nil { return nil } @@ -221,7 +221,7 @@ func NewEngineSubscriptionOnStartHook(fn func(ctx SubscriptionOnStartHookContext } } -type StreamReceiveEventHookContext interface { +type StreamReceiveEventHandlerContext interface { // Request is the initial client request that started the subscription Request() *http.Request // Logger is the logger for the request @@ -234,17 +234,17 @@ type StreamReceiveEventHookContext interface { SubscriptionEventConfiguration() datasource.SubscriptionEventConfiguration } -type StreamReceiveEventHook interface { +type StreamReceiveEventHandler interface { // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the // client. So for a single batch of events received from the provider, this hook will be called one time for each // active subscription. // It is important to optimize the logic inside this hook to avoid performance issues. // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a // StreamHookError. - OnReceiveEvents(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + OnReceiveEvents(ctx StreamReceiveEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } -type StreamPublishEventHookContext interface { +type StreamPublishEventHandlerContext interface { // Request is the original request received by the router. Request() *http.Request // Logger is the logger for the request @@ -257,14 +257,14 @@ type StreamPublishEventHookContext interface { PublishEventConfiguration() datasource.PublishEventConfiguration } -type StreamPublishEventHook interface { +type StreamPublishEventHandler interface { // OnPublishEvents is called each time a batch of events is going to be sent to the provider // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a // StreamHookError. - OnPublishEvents(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) + OnPublishEvents(ctx StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) } -func NewPubSubOnPublishEventsHook(fn func(ctx StreamPublishEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnPublishEventsFn { +func NewPubSubOnPublishEventsHook(fn func(ctx StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnPublishEventsFn { if fn == nil { return nil } @@ -311,7 +311,7 @@ func (c *pubSubStreamReceiveEventHookContext) SubscriptionEventConfiguration() d return c.subscriptionEventConfiguration } -func NewPubSubOnReceiveEventsHook(fn func(ctx StreamReceiveEventHookContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnReceiveEventsFn { +func NewPubSubOnReceiveEventsHook(fn func(ctx StreamReceiveEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error)) datasource.OnReceiveEventsFn { if fn == nil { return nil } From 57bb4d9ff546517a2eee9a7901b54f1a8b6f2088 Mon Sep 17 00:00:00 2001 From: Dominik Korittki <23359034+dkorittki@users.noreply.github.com> Date: Wed, 1 Oct 2025 10:18:14 +0200 Subject: [PATCH 170/173] chore: go mod tidy --- router/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/router/go.sum b/router/go.sum index 6d18d30acb..1a0bc0afe5 100644 --- a/router/go.sum +++ b/router/go.sum @@ -321,8 +321,6 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.229 h1:VCfCX/xmpBGQLhTHJMHLugzJrXJk/smjLRAEruCI0HY= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.229/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.229.0.20250930144208-ddc652f78bbb h1:stBTAle5FyytsTNxYeCwNzYlyhKzlS4he6f7/y6O3qE= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.229.0.20250930144208-ddc652f78bbb/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= From f863f146a435ff65443a1a557ad1624d49c67787 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Wed, 8 Oct 2025 17:42:21 +0200 Subject: [PATCH 171/173] chore: remove last references to CloseSubscription, fix PR comments --- adr/cosmo-streams-v1.md | 26 ++-- router-tests/events/utils.go | 2 +- router-tests/modules/stream_publish_test.go | 1 + router-tests/modules/stream_receive_test.go | 14 +- router/pkg/pubsub/datasource/error.go | 4 - .../pubsub/datasource/pubsubprovider_test.go | 6 +- .../datasource/subscription_event_updater.go | 7 +- .../subscription_event_updater_test.go | 134 ++++++------------ router/pkg/pubsub/redis/engine_datasource.go | 6 +- 9 files changed, 67 insertions(+), 133 deletions(-) diff --git a/adr/cosmo-streams-v1.md b/adr/cosmo-streams-v1.md index 01ae115c25..21b035ff0b 100644 --- a/adr/cosmo-streams-v1.md +++ b/adr/cosmo-streams-v1.md @@ -33,12 +33,6 @@ const ( ProviderTypeRedis ProviderType = "redis" } -// StreamHandlerError is used to customize the error messages and the behavior -type StreamHandlerError struct { - HttpError core.HttpError - CloseSubscription bool -} - // OperationContext already exists, we just have to add the Variables() method type OperationContext interface { Name() string @@ -86,7 +80,7 @@ type SubscriptionOnStartHandlerContext interface { type SubscriptionOnStartHandler interface { // OnSubscriptionOnStart is called once at subscription start - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHandlerError. + // Returning an error will result in a GraphQL error being returned to the client SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) error } @@ -107,7 +101,7 @@ type StreamReceiveEventHandler interface { // OnReceiveEvents is called each time a batch of events is received from the provider before delivering them to the client // So for a single batch of events received from the provider, this hook will be called one time for each active subscription. // It is important to optimize the logic inside this hook to avoid performance issues. - // Returning an error will result in a GraphQL error being returned to the client, could be customized returning a StreamHandlerError. + // Returning an error will result in a GraphQL error being returned to the client OnReceiveEvents(ctx StreamReceiveEventHandlerContext, events []StreamEvent) ([]StreamEvent, error) } @@ -354,11 +348,9 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) er // check if the client is authenticated if ctx.Authentication() == nil { // if the client is not authenticated, return an error - return &StreamHandlerError{ - HttpError: core.HttpError{ - Code: http.StatusUnauthorized, - Message: "client is not authenticated", - }, + return &core.HttpError{ + Code: http.StatusUnauthorized, + Message: "client is not authenticated", CloseSubscription: true, } } @@ -366,11 +358,9 @@ func (m *MyModule) SubscriptionOnStart(ctx SubscriptionOnStartHandlerContext) er // check if the client is allowed to subscribe to the stream clientAllowedEntitiesIds, found := ctx.Authentication().Claims()["readEmployee"] if !found { - return &StreamHandlerError{ - HttpError: core.HttpError{ - Code: http.StatusForbidden, - Message: "client is not allowed to read employees", - }, + return &core.HttpError{ + Code: http.StatusForbidden, + Message: "client is not allowed to read employees", CloseSubscription: true, } } diff --git a/router-tests/events/utils.go b/router-tests/events/utils.go index 95705efb58..b8619c3368 100644 --- a/router-tests/events/utils.go +++ b/router-tests/events/utils.go @@ -40,7 +40,7 @@ func ProduceKafkaMessage(t *testing.T, xEnv *testenv.Environment, timeout time.D xEnv.KafkaClient.Produce(ctx, &kgo.Record{ Topic: xEnv.GetPubSubName(topicName), Value: []byte(message), - }, func(record *kgo.Record, err error) { + }, func(_ *kgo.Record, err error) { pErrCh <- err }) diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 747e85b2e5..74c4a16d98 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -138,6 +138,7 @@ func TestPublishHook(t *testing.T) { Query: `mutation { updateEmployeeMyKafka(employeeID: 3, update: {name: "name test"}) { success } }`, }) require.JSONEq(t, `{"data": {"updateEmployeeMyKafka": {"success": false}}}`, resOne.Body) + require.Equal(t, resOne.Response.StatusCode, 200) requestLog := xEnv.Observer().FilterMessage("Publish Hook has been run") assert.Len(t, requestLog.All(), 1) diff --git a/router-tests/modules/stream_receive_test.go b/router-tests/modules/stream_receive_test.go index 13b63bea96..a1658dc35c 100644 --- a/router-tests/modules/stream_receive_test.go +++ b/router-tests/modules/stream_receive_test.go @@ -23,18 +23,6 @@ import ( "go.uber.org/zap/zapcore" ) -type errorWithCloseSubscription struct { - err error -} - -func (e *errorWithCloseSubscription) Error() string { - return e.err.Error() -} - -func (e *errorWithCloseSubscription) CloseSubscription() bool { - return true -} - func TestReceiveHook(t *testing.T) { t.Parallel() @@ -465,7 +453,7 @@ func TestReceiveHook(t *testing.T) { Modules: map[string]interface{}{ "streamReceiveModule": stream_receive.StreamReceiveModule{ Callback: func(ctx core.StreamReceiveEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - return nil, &errorWithCloseSubscription{err: errors.New("test error from streamevents hook")} + return nil, errors.New("test error from streamevents hook") }, }, }, diff --git a/router/pkg/pubsub/datasource/error.go b/router/pkg/pubsub/datasource/error.go index b6f4b449f9..f09b271688 100644 --- a/router/pkg/pubsub/datasource/error.go +++ b/router/pkg/pubsub/datasource/error.go @@ -15,7 +15,3 @@ func NewError(publicMsg string, cause error) *Error { Internal: cause, } } - -type ErrorWithCloseSubscription interface { - CloseSubscription() bool -} \ No newline at end of file diff --git a/router/pkg/pubsub/datasource/pubsubprovider_test.go b/router/pkg/pubsub/datasource/pubsubprovider_test.go index 2ff1ed57da..6ef41c56a5 100644 --- a/router/pkg/pubsub/datasource/pubsubprovider_test.go +++ b/router/pkg/pubsub/datasource/pubsubprovider_test.go @@ -1,6 +1,7 @@ package datasource import ( + "bytes" "context" "errors" "testing" @@ -20,8 +21,9 @@ func (e *testEvent) GetData() []byte { } func (e *testEvent) Clone() StreamEvent { - e2 := *e - return &e2 + return &testEvent{ + data: bytes.Clone(e.data), + } } type testSubscriptionConfig struct { diff --git a/router/pkg/pubsub/datasource/subscription_event_updater.go b/router/pkg/pubsub/datasource/subscription_event_updater.go index 82ca60007e..95289bb313 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater.go @@ -55,11 +55,8 @@ func (s *subscriptionEventUpdater) Update(events []StreamEvent) { zap.String("field_name", s.subscriptionEventConfiguration.RootFieldName()), ) } - // Check if the error is a StreamHookError and should close the subscription - // We use type assertion to check for the CloseSubscription method without importing core - if hookErr, ok := err.(ErrorWithCloseSubscription); ok && hookErr.CloseSubscription() { - s.eventUpdater.CloseSubscription(resolve.SubscriptionCloseKindNormal, subId) - } + // Always close the subscription when a hook reports an error to avoid inconsistent state. + s.eventUpdater.CloseSubscription(resolve.SubscriptionCloseKindNormal, subId) } } } diff --git a/router/pkg/pubsub/datasource/subscription_event_updater_test.go b/router/pkg/pubsub/datasource/subscription_event_updater_test.go index 388561328b..79fd140a51 100644 --- a/router/pkg/pubsub/datasource/subscription_event_updater_test.go +++ b/router/pkg/pubsub/datasource/subscription_event_updater_test.go @@ -130,6 +130,7 @@ func TestSubscriptionEventUpdater_UpdateSubscriptions_WithHooks_Error(t *testing mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ context.Background(): subId, }) + mockUpdater.On("CloseSubscription", resolve.SubscriptionCloseKindNormal, subId).Return() // Should not call Update or UpdateSubscription on eventUpdater since hook fails updater := &subscriptionEventUpdater{ @@ -145,6 +146,7 @@ func TestSubscriptionEventUpdater_UpdateSubscriptions_WithHooks_Error(t *testing // Assert that Update and UpdateSubscription were not called on the eventUpdater mockUpdater.AssertNotCalled(t, "Update") mockUpdater.AssertNotCalled(t, "UpdateSubscription") + mockUpdater.AssertCalled(t, "CloseSubscription", resolve.SubscriptionCloseKindNormal, subId) } func TestSubscriptionEventUpdater_Update_WithMultipleHooks_Success(t *testing.T) { @@ -529,89 +531,57 @@ func TestSubscriptionEventUpdater_Close_WithDifferentCloseKinds(t *testing.T) { } } -func TestSubscriptionEventUpdater_UpdateSubscription_WithStreamHookError_CloseSubscription(t *testing.T) { - mockUpdater := NewMockSubscriptionUpdater(t) - config := &testSubscriptionEventConfig{ - providerID: "test-provider", - providerType: ProviderTypeNats, - fieldName: "testField", - } - events := []StreamEvent{ - &testEvent{data: []byte("test data")}, - } - - // Create a mock StreamHookError with CloseSubscription=true - mockHookError := &mockStreamHookError{ - closeSubscription: true, - message: "subscription should close", - } - - // Define hook that returns a StreamHookError with CloseSubscription=true - testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { - return events, mockHookError - } - - updater := &subscriptionEventUpdater{ - eventUpdater: mockUpdater, - subscriptionEventConfiguration: config, - hooks: Hooks{ - OnReceiveEvents: []OnReceiveEventsFn{testHook}, +func TestSubscriptionEventUpdater_UpdateSubscription_WithHookError_ClosesSubscription(t *testing.T) { + testCases := []struct { + name string + hookError error + }{ + { + name: "generic error", + hookError: errors.New("subscription should close"), + }, + { + name: "error implementing CloseSubscription false", + hookError: errors.New("subscription should still close"), }, } - // Expect call to UpdateSubscription with modified data - subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} - mockUpdater.On("UpdateSubscription", subId, []byte("test data")).Return() - mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ - context.Background(): subId, - }) - mockUpdater.On("CloseSubscription", resolve.SubscriptionCloseKindNormal, subId).Return() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockUpdater := NewMockSubscriptionUpdater(t) + config := &testSubscriptionEventConfig{ + providerID: "test-provider", + providerType: ProviderTypeNats, + fieldName: "testField", + } + events := []StreamEvent{ + &testEvent{data: []byte("test data")}, + } - updater.Update(events) -} + testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { + return events, tc.hookError + } -func TestSubscriptionEventUpdater_UpdateSubscription_WithStreamHookError_NoCloseSubscription(t *testing.T) { - mockUpdater := NewMockSubscriptionUpdater(t) - config := &testSubscriptionEventConfig{ - providerID: "test-provider", - providerType: ProviderTypeNats, - fieldName: "testField", - } - events := []StreamEvent{ - &testEvent{data: []byte("test data")}, - } + updater := &subscriptionEventUpdater{ + eventUpdater: mockUpdater, + subscriptionEventConfiguration: config, + hooks: Hooks{ + OnReceiveEvents: []OnReceiveEventsFn{testHook}, + }, + } - // Create a mock StreamHookError with CloseSubscription=false - mockHookError := &mockStreamHookError{ - closeSubscription: false, - message: "subscription should not close", - } + subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} + mockUpdater.On("UpdateSubscription", subId, []byte("test data")).Return() + mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ + context.Background(): subId, + }) + mockUpdater.On("CloseSubscription", resolve.SubscriptionCloseKindNormal, subId).Return() - // Define hook that returns a StreamHookError with CloseSubscription=false - testHook := func(ctx context.Context, cfg SubscriptionEventConfiguration, events []StreamEvent) ([]StreamEvent, error) { - return events, mockHookError - } + updater.Update(events) - updater := &subscriptionEventUpdater{ - eventUpdater: mockUpdater, - subscriptionEventConfiguration: config, - hooks: Hooks{ - OnReceiveEvents: []OnReceiveEventsFn{testHook}, - }, + mockUpdater.AssertCalled(t, "CloseSubscription", resolve.SubscriptionCloseKindNormal, subId) + }) } - - // Expect call to UpdateSubscription with modified data - subId := resolve.SubscriptionIdentifier{ConnectionID: 1, SubscriptionID: 1} - mockUpdater.On("UpdateSubscription", subId, []byte("test data")).Return() - mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ - context.Background(): subId, - }) - - updater.Update(events) - - // Assert that Update was not called on the eventUpdater - mockUpdater.AssertNotCalled(t, "Update") - mockUpdater.AssertNotCalled(t, "CloseSubscription") } func TestSubscriptionEventUpdater_UpdateSubscription_WithHooks_Error_LoggerWritesError(t *testing.T) { @@ -644,26 +614,14 @@ func TestSubscriptionEventUpdater_UpdateSubscription_WithHooks_Error_LoggerWrite mockUpdater.On("Subscriptions").Return(map[context.Context]resolve.SubscriptionIdentifier{ context.Background(): subId, }) + mockUpdater.On("CloseSubscription", resolve.SubscriptionCloseKindNormal, subId).Return() updater.Update(events) // Assert that Update was not called on the eventUpdater mockUpdater.AssertNotCalled(t, "UpdateSubscription") + mockUpdater.AssertCalled(t, "CloseSubscription", resolve.SubscriptionCloseKindNormal, subId) msgs := logObserver.FilterMessageSnippet("An error occurred while processing stream events hooks").TakeAll() assert.Equal(t, 1, len(msgs)) } - -// mockStreamHookError implements the CloseSubscription() method for testing -type mockStreamHookError struct { - closeSubscription bool - message string -} - -func (e *mockStreamHookError) Error() string { - return e.message -} - -func (e *mockStreamHookError) CloseSubscription() bool { - return e.closeSubscription -} diff --git a/router/pkg/pubsub/redis/engine_datasource.go b/router/pkg/pubsub/redis/engine_datasource.go index f57e9501cd..e796b60e66 100644 --- a/router/pkg/pubsub/redis/engine_datasource.go +++ b/router/pkg/pubsub/redis/engine_datasource.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "slices" "github.com/buger/jsonparser" "github.com/cespare/xxhash/v2" @@ -24,8 +25,9 @@ func (e *Event) GetData() []byte { } func (e *Event) Clone() datasource.StreamEvent { - e2 := *e - return &e2 + return &Event{ + Data: slices.Clone(e.Data), + } } // SubscriptionEventConfiguration contains configuration for subscription events From b975c6037ad357275eb4d87a90e6711fed15c95d Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 9 Oct 2025 10:22:36 +0200 Subject: [PATCH 172/173] chore: replace StreamHandlerError with already existing httpGraphqlError --- .../modules/start_subscription_test.go | 6 ++-- router-tests/modules/stream_publish_test.go | 8 ++--- router/core/errors.go | 8 ++--- router/core/graphql_handler.go | 16 ++++----- router/core/subscriptions_modules.go | 36 ------------------- 5 files changed, 19 insertions(+), 55 deletions(-) diff --git a/router-tests/modules/start_subscription_test.go b/router-tests/modules/start_subscription_test.go index 3cb82a4f76..b9d5e2f0ac 100644 --- a/router-tests/modules/start_subscription_test.go +++ b/router-tests/modules/start_subscription_test.go @@ -181,7 +181,7 @@ func TestStartSubscriptionHook(t *testing.T) { "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHandlerContext) error { callbackCalled <- true - return core.NewStreamHookError(nil, "subscription closed", http.StatusOK, "") + return core.NewHttpGraphqlError("subscription closed", http.StatusText(http.StatusOK), http.StatusOK) }, }, }, @@ -366,7 +366,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHandlerContext) error { - return core.NewStreamHookError(errors.New("test error"), "test error", http.StatusLoopDetected, http.StatusText(http.StatusLoopDetected)) + return core.NewHttpGraphqlError("test error", http.StatusText(http.StatusLoopDetected), http.StatusLoopDetected) }, }, }, @@ -594,7 +594,7 @@ func TestStartSubscriptionHook(t *testing.T) { Modules: map[string]interface{}{ "startSubscriptionModule": start_subscription.StartSubscriptionModule{ Callback: func(ctx core.SubscriptionOnStartHandlerContext) error { - return core.NewStreamHookError(errors.New("subscription closed"), "subscription closed", http.StatusOK, "NotFound") + return core.NewHttpGraphqlError("subscription closed", http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) }, CallbackOnOriginResponse: func(response *http.Response, ctx core.RequestContext) *http.Response { originResponseCalled <- response diff --git a/router-tests/modules/stream_publish_test.go b/router-tests/modules/stream_publish_test.go index 74c4a16d98..6fb7485dc3 100644 --- a/router-tests/modules/stream_publish_test.go +++ b/router-tests/modules/stream_publish_test.go @@ -2,7 +2,7 @@ package module_test import ( "encoding/json" - "errors" + "net/http" "strconv" "testing" "time" @@ -115,7 +115,7 @@ func TestPublishHook(t *testing.T) { Modules: map[string]interface{}{ "publishModule": stream_publish.PublishModule{ Callback: func(ctx core.StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR") + return events, core.NewHttpGraphqlError("test", http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) }, }, }, @@ -160,7 +160,7 @@ func TestPublishHook(t *testing.T) { Modules: map[string]interface{}{ "publishModule": stream_publish.PublishModule{ Callback: func(ctx core.StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR") + return events, core.NewHttpGraphqlError("test", http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) }, }, }, @@ -214,7 +214,7 @@ func TestPublishHook(t *testing.T) { Modules: map[string]interface{}{ "publishModule": stream_publish.PublishModule{ Callback: func(ctx core.StreamPublishEventHandlerContext, events []datasource.StreamEvent) ([]datasource.StreamEvent, error) { - return events, core.NewStreamHookError(errors.New("test"), "test", 500, "INTERNAL_SERVER_ERROR") + return events, core.NewHttpGraphqlError("test", http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) }, }, }, diff --git a/router/core/errors.go b/router/core/errors.go index 07f3175f64..2ce688bbef 100644 --- a/router/core/errors.go +++ b/router/core/errors.go @@ -35,7 +35,7 @@ const ( errorTypeInvalidWsSubprotocol errorTypeEDFSInvalidMessage errorTypeMergeResult - errorTypeStreamHookError + errorTypeHttpError ) type ( @@ -90,9 +90,9 @@ func getErrorType(err error) errorType { if errors.As(err, &mergeResultErr) { return errorTypeMergeResult } - var streamHandlerErr *StreamHandlerError - if errors.As(err, &streamHandlerErr) { - return errorTypeStreamHookError + var httpError *httpGraphqlError + if errors.As(err, &httpError) { + return errorTypeHttpError } return errorTypeUnknown } diff --git a/router/core/graphql_handler.go b/router/core/graphql_handler.go index 58a9ba245c..845b8bdac0 100644 --- a/router/core/graphql_handler.go +++ b/router/core/graphql_handler.go @@ -400,21 +400,21 @@ func (h *GraphQLHandler) WriteError(ctx *resolve.Context, err error, res *resolv if isHttpResponseWriter { httpWriter.WriteHeader(http.StatusInternalServerError) } - case errorTypeStreamHookError: - var streamHookErr *StreamHandlerError - if !errors.As(err, &streamHookErr) { + case errorTypeHttpError: + var httpErr *httpGraphqlError + if !errors.As(err, &httpErr) { response.Errors[0].Message = "Internal server error" return } - response.Errors[0].Message = streamHookErr.Message() - if streamHookErr.Code() != "" || streamHookErr.StatusCode() != 0 { + response.Errors[0].Message = httpErr.Message() + if httpErr.ExtensionCode() != "" || httpErr.StatusCode() != 0 { response.Errors[0].Extensions = &Extensions{ - Code: streamHookErr.Code(), - StatusCode: streamHookErr.StatusCode(), + Code: httpErr.ExtensionCode(), + StatusCode: httpErr.StatusCode(), } } if isHttpResponseWriter { - httpWriter.WriteHeader(streamHookErr.StatusCode()) + httpWriter.WriteHeader(httpErr.StatusCode()) } } diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 28f1d59509..6a9ba7e890 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -11,42 +11,6 @@ import ( "go.uber.org/zap" ) -// StreamHandlerError is used to customize the error messages and the behavior -type StreamHandlerError struct { - err error - message string - statusCode int - code string -} - -func (e *StreamHandlerError) Error() string { - if e.err != nil { - return e.err.Error() - } - return e.message -} - -func (e *StreamHandlerError) Message() string { - return e.message -} - -func (e *StreamHandlerError) StatusCode() int { - return e.statusCode -} - -func (e *StreamHandlerError) Code() string { - return e.code -} - -func NewStreamHookError(err error, message string, statusCode int, code string) *StreamHandlerError { - return &StreamHandlerError{ - err: err, - message: message, - statusCode: statusCode, - code: code, - } -} - type SubscriptionOnStartHandlerContext interface { // Request is the original request received by the router. Request() *http.Request From ce549277bf797ddf6ca6b399a0c39179065b1171 Mon Sep 17 00:00:00 2001 From: Alessandro Pagnin Date: Thu, 9 Oct 2025 12:26:31 +0200 Subject: [PATCH 173/173] chore: wrong clone in the EngineEvent --- router/core/subscriptions_modules.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/router/core/subscriptions_modules.go b/router/core/subscriptions_modules.go index 6a9ba7e890..e3279c811d 100644 --- a/router/core/subscriptions_modules.go +++ b/router/core/subscriptions_modules.go @@ -3,6 +3,7 @@ package core import ( "context" "net/http" + "slices" "github.com/wundergraph/cosmo/router/pkg/authentication" "github.com/wundergraph/cosmo/router/pkg/pubsub/datasource" @@ -100,8 +101,9 @@ func (e *EngineEvent) GetData() []byte { } func (e *EngineEvent) Clone() datasource.StreamEvent { - e2 := *e - return &e2 + return &EngineEvent{ + Data: slices.Clone(e.Data), + } } type engineSubscriptionOnStartHookContext struct {