Skip to content

[Streams] Allow GroupBy with infinite output substreams #7607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Aaronontheweb merged 2 commits into from
Apr 28, 2025
Merged

[Streams] Allow GroupBy with infinite output substreams #7607

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a04da94
[Streams] Allow GroupBy with infinite output substreams
Arkatufus Apr 28, 2025
4ccc949
Add public API oveloads
Arkatufus Apr 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/core/Akka.Streams.Tests/Dsl/FlowGroupBySpec.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,29 @@ await this.AssertAllStagesStoppedAsync(async () =>
}, Materializer);
}

[Fact(DisplayName = "GroupBy must not have substream limit when maxSubStream is set to -1")]
public async Task GroupBy_UnlimitedSubstreamTest()
{
await this.AssertAllStagesStoppedAsync(async () =>
{
var f = Flow.Create<int>().GroupBy(-1, x => x).PrefixAndTail(0).MergeSubstreams();
var (up, down) = ((Flow<int, (IImmutableList<int>, Source<int, NotUsed>), NotUsed>)f)
.RunWith(this.SourceProbe<int>(), this.SinkProbe<(IImmutableList<int>, Source<int, NotUsed>)>(), Materializer);

await down.RequestAsync(100);

foreach (var i in Enumerable.Range(0, 100))
{
await up.SendNextAsync(i);
var (_, source) = await down.ExpectNextAsync();
var (sub, probe) = await StreamPuppet(source.RunWith(Sink.AsPublisher<int>(false), Materializer), this);

sub.Request(1);
await probe.ExpectNextAsync(i);
}
}, Materializer);
}

[Fact]
public async Task GroupBy_must_resume_when_exceeding_maxSubStreams()
{
Expand Down
9 changes: 8 additions & 1 deletion src/core/Akka.Streams/Dsl/FlowOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1346,7 +1346,7 @@ public static Flow<TIn, TOut2, TMat> Transform<TIn, TOut1, TOut2, TMat>(this Flo
/// <typeparam name="TMat">TBD</typeparam>
/// <typeparam name="TKey">TBD</typeparam>
/// <param name="flow">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
/// <param name="allowClosedSubstreamRecreation">Enables recreation of already closed substreams if elements with their corresponding keys arrive after completion</param>
/// <returns>TBD</returns>
Expand All @@ -1366,6 +1366,13 @@ public static SubFlow<TOut, TMat, Sink<TIn, TMat>> GroupBy<TIn, TOut, TMat, TKey
/// </para>
/// See <seealso cref="GroupBy{TIn, TOut, TMat, TKey}(Flow{TIn, TOut, TMat}, int, Func{TOut, TKey}, bool)"/>
/// </summary>
/// <typeparam name="TIn">TBD</typeparam>
/// <typeparam name="TOut">TBD</typeparam>
/// <typeparam name="TMat">TBD</typeparam>
/// <typeparam name="TKey">TBD</typeparam>
/// <param name="flow">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
public static SubFlow<TOut, TMat, Sink<TIn, TMat>> GroupBy<TIn, TOut, TMat, TKey>(this Flow<TIn, TOut, TMat> flow, int maxSubstreams, Func<TOut, TKey> groupingFunc) =>
flow.GroupBy(maxSubstreams, groupingFunc, (f, s) => ((Flow<TIn, Source<TOut, NotUsed>, TMat>)f).To(s), false);

Expand Down
18 changes: 9 additions & 9 deletions src/core/Akka.Streams/Dsl/Internal/InternalFlowOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1314,7 +1314,7 @@ public static IFlow<TOut, TMat> Transform<TIn, TOut, TMat>(this IFlow<TIn, TMat>
/// <typeparam name="TKey">TBD</typeparam>
/// <typeparam name="TClosed">TBD</typeparam>
/// <param name="flow">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
/// <param name="toFunc">TBD</param>
/// <param name="allowClosedSubstreamRecreation">Enables recreation of already closed substreams if elements with their corresponding keys arrive after completion</param>
Expand All @@ -1328,14 +1328,14 @@ public static SubFlow<T, TMat, TClosed> GroupBy<T, TMat, TKey, TClosed>(
{
var merge = new GroupByMergeBack<T, TMat, TKey>(flow, maxSubstreams, groupingFunc, allowClosedSubstreamRecreation);

TClosed finish(Sink<T, TMat> s)
return new SubFlowImpl<T, T, TMat, TClosed>(Flow.Create<T, TMat>(), merge, Finish);

TClosed Finish(Sink<T, TMat> s)
{
return toFunc(
flow.Via(new Fusing.GroupBy<T, TKey>(maxSubstreams, groupingFunc, allowClosedSubstreamRecreation)),
Sink.ForEach<Source<T, NotUsed>>(e => e.RunWith(s, Fusing.GraphInterpreter.Current.Materializer)));
}

return new SubFlowImpl<T, T, TMat, TClosed>(Flow.Create<T, TMat>(), merge, finish);
}

/// <summary>
Expand All @@ -1354,8 +1354,8 @@ TClosed finish(Sink<T, TMat> s)
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TClosed"></typeparam>
/// <param name="flow"></param>
/// <param name="maxSubstreams"></param>
/// <param name="groupingFunc"></param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
/// <param name="toFunc"></param>
/// <returns></returns>
public static SubFlow<T, TMat, TClosed> GroupBy<T, TMat, TKey, TClosed>(
Expand All @@ -1381,9 +1381,9 @@ internal sealed class GroupByMergeBack<TOut, TMat, TKey> : IMergeBack<TOut, TMat
/// TBD
/// </summary>
/// <param name="self">TBD</param>
/// <param name="maxSubstreams">TBD</param>
/// <param name="groupingFunc">TBD</param>
/// <param name="allowClosedSubstreamRecreation">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
/// <param name="allowClosedSubstreamRecreation">Enables recreation of already closed substreams if elements with their corresponding keys arrive after completion</param>
public GroupByMergeBack(IFlow<TOut, TMat> self, int maxSubstreams, Func<TOut, TKey> groupingFunc, bool allowClosedSubstreamRecreation = false)
{
_self = self;
Expand Down
4 changes: 2 additions & 2 deletions src/core/Akka.Streams/Dsl/SourceOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1215,8 +1215,8 @@ public static Source<TOut2, TMat> Transform<TOut1, TOut2, TMat>(this Source<TOut
/// <typeparam name="TMat">TBD</typeparam>
/// <typeparam name="TKey">TBD</typeparam>
/// <param name="flow">TBD</param>
/// <param name="maxSubstreams">TBD</param>
/// <param name="groupingFunc">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="groupingFunc">Computes the key for each element</param>
/// <returns>TBD</returns>
public static SubFlow<TOut, TMat, IRunnableGraph<TMat>> GroupBy<TOut, TMat, TKey>(this Source<TOut, TMat> flow, int maxSubstreams, Func<TOut, TKey> groupingFunc)
{
Expand Down
13 changes: 8 additions & 5 deletions src/core/Akka.Streams/Implementation/Fusing/StreamOfStreams.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,9 +419,9 @@ public void OnPush()
}
else
{
if (_activeSubstreams.Count + _closedSubstreams.Count == _stage._maxSubstreams)
if (_stage._maxSubstreams > -1 && _activeSubstreams.Count + _closedSubstreams.Count == _stage._maxSubstreams)

@Arkatufus Arkatufus Apr 28, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actual fix. Only check substream overflow condition if _maxSubstream is greater than or equals to 0.

throw new TooManySubstreamsOpenException();
else if (_closedSubstreams.Contains(key) && !HasBeenPulled(_stage.In))
if (_closedSubstreams.Contains(key) && !HasBeenPulled(_stage.In))
Pull(_stage.In);
else
RunSubstream(key, element);
Expand Down Expand Up @@ -640,11 +640,14 @@ public void OnDownstreamFinish(Exception cause)
/// <summary>
/// TBD
/// </summary>
/// <param name="maxSubstreams">TBD</param>
/// <param name="keyFor">TBD</param>
/// <param name="allowClosedSubstreamRecreation">TBD</param>
/// <param name="maxSubstreams">Configures the maximum number of substreams (keys) that are supported; if more distinct keys are encountered then the stream fails. Set to -1 for infinite substreams.</param>
/// <param name="keyFor">Computes the key for each element</param>
/// <param name="allowClosedSubstreamRecreation">Enables recreation of already closed substreams if elements with their corresponding keys arrive after completion</param>
public GroupBy(int maxSubstreams, Func<T, TKey> keyFor, bool allowClosedSubstreamRecreation = false)
{
if (maxSubstreams < -1)
throw new ArgumentException("maxSubstreams must be greater or equals to -1.", nameof(maxSubstreams));

_maxSubstreams = maxSubstreams;
_keyFor = keyFor;
_allowClosedSubstreamRecreation = allowClosedSubstreamRecreation;
Expand Down