fix: bound profiler memory growth by trimming interned call stacks

src/Sentry.Profiling/SampleProfileBuilder.cs

@@ -11,8 +11,12 @@ namespace Sentry.Profiling;
 internal class SampleProfileBuilder
 {
     private readonly SentryOptions _options;
+    private readonly SampleProfilerSession? _session;
     private readonly TraceLog _traceLog;
 
+    // The trim generation this builder's _stackIndexes cache was populated against.
+    private int _trimGeneration;
+
     // Output profile being built.
     public readonly SampleProfile Profile = new();
 
@@ -32,14 +36,30 @@ internal class SampleProfileBuilder
     // TODO make downsampling conditional once this is available: https://github.com/dotnet/runtime/issues/82939
     private readonly Downsampler _downsampler = new();
 
+    // For a TraceLog read from a file, where nothing trims it and no generation tracking is needed.
     public SampleProfileBuilder(SentryOptions options, TraceLog traceLog)
     {
         _options = options;
         _traceLog = traceLog;
     }
 
+    public SampleProfileBuilder(SentryOptions options, SampleProfilerSession session)
+        : this(options, session.TraceLog)
+    {
+        _session = session;
+        _trimGeneration = session.TrimGeneration;
+    }
+
     internal void AddSample(TraceEvent data, double timestampMs)
     {
+        // The interning tables have been discarded since we last looked so we have to
+        // invalidate the _stackIndexes cache (means stacks get walked again)
+        if (_session is { } session && _trimGeneration != session.TrimGeneration)
+        {
+            _trimGeneration = session.TrimGeneration;
+            _stackIndexes.Clear();
+        }
+
         var thread = data.Thread();
         if (thread is null || thread.ThreadIndex == ThreadIndex.Invalid)
         {

src/Sentry.Profiling/SampleProfilerSession.cs

@@ -56,6 +56,20 @@ private SampleProfilerSession(SentryStopwatch stopwatch, EventPipeSession sessio
 
     public TraceLog TraceLog => EventSource.TraceLog;
 
+    // Bumped every time the interning tables are discarded. Anything caching a 
+    // CallStackIndex can invalidate it's cache whenever this changes
+    internal int TrimGeneration;
+
+    /// <summary>
+    /// Discards TraceLog's call stack interning tables, invalidating CallStackIndexes.
+    /// Must be called from the event processing thread.
+    /// </summary>
+    internal void TrimLiveSessionState()
+    {
+        TraceLog.TrimLiveSessionState();
+        TrimGeneration++;
+    }
+
     private static InterlockedBoolean _throwOnNextStartupForTests = false;
 
     internal static bool ThrowOnNextStartupForTests

src/Sentry.Profiling/SamplingTransactionProfiler.cs

@@ -24,7 +24,7 @@ public SamplingTransactionProfiler(SentryOptions options, SampleProfilerSession
         _cancellationToken = cancellationToken;
         _startTimeMs = session.Elapsed.TotalMilliseconds;
         _endTimeMs = double.MaxValue;
-        _processor = new SampleProfileBuilder(options, session.TraceLog);
+        _processor = new SampleProfileBuilder(options, session);
         session.SampleEventParser.ThreadSample += OnThreadSample;
         cancellationToken.Register(() =>
         {

src/Sentry.Profiling/SamplingTransactionProfilerFactory.cs

@@ -1,3 +1,4 @@
+using Microsoft.Diagnostics.Tracing;
 using Sentry.Extensibility;
 using Sentry.Internal;
 
@@ -14,10 +15,17 @@ internal class SamplingTransactionProfilerFactory : IDisposable, ITransactionPro
     // Stop profiling after the given number of milliseconds.
     private const int TIME_LIMIT_MS = 30_000;
 
+    // Once the interning tables exceed this many entries we discard them ASAP
+    internal static int MaxCallStackCount = 100_000;
+
     private readonly SentryOptions _options;
 
     internal Task<SampleProfilerSession> _sessionTask;
 
+    private volatile SampleProfilerSession? _session;
+
+    internal int TrimCount;
+
     private bool _errorLogged = false;
 
     public SamplingTransactionProfilerFactory(SentryOptions options, TimeSpan startupTimeout)
@@ -32,6 +40,9 @@ public SamplingTransactionProfilerFactory(SentryOptions options, TimeSpan startu
             // This can block indefinitely.
             await session.WaitForFirstEventAsync().ConfigureAwait(false);
 
+            _session = session;
+            session.SampleEventParser.ThreadSample += TrimSessionStateIfNeeded;
+
             return session;
         });
 
@@ -81,6 +92,39 @@ public SamplingTransactionProfilerFactory(SentryOptions options, TimeSpan startu
         return null;
     }
 
+    /// <summary>
+    /// Discards TraceLog's call stack interning tables once they grow past <see cref="MaxCallStackCount"/>.
+    /// <para>
+    /// Runs for every sample, on the session's event processing thread - which is the only thread
+    /// allowed to trim, so this is where it has to happen. It cannot be driven off profile completion
+    /// alone, because the tables grow whether or not a profile is running.
+    /// </para>
+    /// <para>
+    /// Safe to do mid-profile: SampleProfileBuilder keys a cache off CallStackIndex, which a trim
+    /// reissues from zero, but it notices the bumped generation and drops that cache.
+    /// </para>
+    /// </summary>
+    private void TrimSessionStateIfNeeded(TraceEvent data)
+    {
+        var session = _session;
+        if (session is null || session.TraceLog.CallStacks.Count <= MaxCallStackCount)
+        {
+            return;
+        }
+
+        try
+        {
+            _options.LogDebug("Trimming profiler session state, {0} interned call stacks.", session.TraceLog.CallStacks.Count);
+            session.TrimLiveSessionState();
+            TrimCount++;
+        }
+        catch (Exception e)
+        {
+            // Never let this take down event processing for the whole session.
+            _options.LogWarning(e, "Failed to trim profiler session state.");
+        }
+    }
+
     public void Dispose()
     {
         _sessionTask.ContinueWith(session => session.Dispose());

test/Sentry.Profiling.Tests/Resources/README.md

@@ -49,3 +49,27 @@ And for reference, you can create a JSON that can be displayed by [SpeedScope](h
 ```shell-script
 dotnet-trace convert sample.nettrace --format Speedscope
 ```
+
+## Regenerating `sample.etlx` after a perfview submodule bump
+
+`sample.etlx` is committed, and its format version is tied to the TraceEvent build in
+`modules/perfview`. A submodule bump can therefore leave it unreadable, which surfaces as:
+
+```
+FastSerialization.SerializationException : File format is version 74 App accepts formats >= 78.
+```
+
+To regenerate it, delete the file and run the tests once — `TraceLogProcessorTests` rebuilds it from
+`sample.nettrace` when it is absent:
+
+```shell-script
+rm test/Sentry.Profiling.Tests/Resources/sample.etlx
+dotnet test test/Sentry.Profiling.Tests --filter "FullyQualifiedName~TraceLogProcessorTests"
+```
+
+Then commit the regenerated file, along with any `*.verified.txt` snapshot changes it produces
+(`pwsh ./scripts/accept-verifier-changes.ps1` — review the diff, module names in particular).
+
+Do **not** leave it uncommitted and rely on regeneration at test time. The path is shared by every
+target framework, and `dotnet test` runs one host per TFM, so they race on both this file and the
+`sample.etlx.new` temp that TraceEvent writes alongside it.

test/Sentry.Profiling.Tests/SamplingTransactionProfilerTests.cs

@@ -457,6 +457,122 @@ public void ProfilerIntegration_WithProfilingEnabled_SetsFactory()
         Assert.NotNull(hub.Options.TransactionProfilerFactory);
     }
 
+    [SkippableFact]
+    public void Profiler_WhenNoProfileRunning_TrimsInternedCallStacks()
+    {
+        Skip.If(TestEnvironment.IsGitHubActions, "Flaky in CI.");
+
+        // The interning tables grow whether or not a profile is running, so this deliberately never
+        // starts one - the trim has to happen off the back of ordinary samples. See #5469.
+        var originalMax = SamplingTransactionProfilerFactory.MaxCallStackCount;
+        SamplingTransactionProfilerFactory.MaxCallStackCount = 100;
+        try
+        {
+            using var factory = new SamplingTransactionProfilerFactory(_testSentryOptions, TimeSpan.FromSeconds(30));
+            SkipIfFailsInCI(() => factory._sessionTask.Wait(60_000));
+            var traceLog = factory._sessionTask.Result.TraceLog;
+
+            // Produce a variety of stack shapes so the sampler has something to intern.
+            var stopwatch = Stopwatch.StartNew();
+            var random = new Random(4242);
+            var highWaterMark = 0;
+            while (stopwatch.ElapsedMilliseconds < 3_000 && factory.TrimCount == 0)
+            {
+                RecursiveWork(random.Next(8, 24), random);
+                highWaterMark = Math.Max(highWaterMark, traceLog.CallStacks.Count);
+            }
+
+            SkipIfFailsInCI(() =>
+            {
+                if (factory.TrimCount == 0)
+                {
+                    throw new Exception($"No trim occurred; call stacks peaked at {highWaterMark}.");
+                }
+            });
+
+            // Having trimmed at least once, the table must have been reset rather than left to grow.
+            Assert.True(factory.TrimCount > 0, $"Expected at least one trim, call stacks peaked at {highWaterMark}.");
+            Assert.True(traceLog.CallStacks.Count < highWaterMark,
+                $"Expected the interning table to shrink after trimming, but it is {traceLog.CallStacks.Count} against a peak of {highWaterMark}.");
+        }
+        finally
+        {
+            SamplingTransactionProfilerFactory.MaxCallStackCount = originalMax;
+        }
+    }
+
+    [SkippableFact]
+    public void Profiler_WhileProfileRunning_StillTrimsInternedCallStacks()
+    {
+        Skip.If(TestEnvironment.IsGitHubActions, "Flaky in CI.");
+
+        // A profile is held open for the whole test. Trimming must still happen: profiles can run
+        // back to back under load, so gating the trim on "no profile running" would starve it
+        // exactly when the tables grow fastest. The profile is then collected to show that trimming
+        // mid-profile does not corrupt its output.
+        var originalMax = SamplingTransactionProfilerFactory.MaxCallStackCount;
+        SamplingTransactionProfilerFactory.MaxCallStackCount = 100;
+        try
+        {
+            using var factory = new SamplingTransactionProfilerFactory(_testSentryOptions, TimeSpan.FromSeconds(30));
+            SkipIfFailsInCI(() => factory._sessionTask.Wait(60_000));
+
+            var clock = SentryStopwatch.StartNew();
+            var transactionTracer = new TransactionTracer(Substitute.For<IHub>(), "test", "");
+            var sut = factory.Start(transactionTracer, CancellationToken.None) as SamplingTransactionProfiler;
+            SkipIfFailsInCI(() => ArgumentNullException.ThrowIfNull(sut));
+            transactionTracer.TransactionProfiler = sut;
+
+            // Discard trims from before the profile started - the tiny budget means the session
+            // startup alone can trigger one. From here every trim happens while _inProgress is true.
+            factory.TrimCount = 0;
+
+            var stopwatch = Stopwatch.StartNew();
+            var random = new Random(4242);
+            while (stopwatch.ElapsedMilliseconds < 5_000 && factory.TrimCount == 0)
+            {
+                RecursiveWork(random.Next(8, 24), random);
+            }
+
+            SkipIfFailsInCI(() =>
+            {
+                if (factory.TrimCount == 0)
+                {
+                    throw new Exception("No trim occurred while a profile was running.");
+                }
+            });
+            Assert.True(factory.TrimCount > 0, "Expected trimming to happen even while a profile is running.");
+
+            sut!.Finish();
+            var elapsedNanoseconds = (ulong)((clock.CurrentDateTimeOffset - clock.StartDateTimeOffset).TotalMilliseconds * 1_000_000);
+            var collectTask = sut.CollectAsync(new SentryTransaction(transactionTracer));
+            collectTask.Wait();
+            ValidateProfile(collectTask.Result.Profile, elapsedNanoseconds);
+        }
+        finally
+        {
+            SamplingTransactionProfilerFactory.MaxCallStackCount = originalMax;
+        }
+    }
+
+    private static long RecursiveWork(int depth, Random random)
+    {
+        if (depth <= 0)
+        {
+            double sink = 0;
+            for (var i = 1; i < 5_000; i++)
+            {
+                sink += Math.Sqrt(i);
+            }
+            return (long)sink;
+        }
+
+        // Branch so that different calls produce different stack shapes.
+        return (random.Next(2) == 0)
+            ? RecursiveWork(depth - 1, random) + depth
+            : RecursiveWork(depth - 1, random) - depth;
+    }
+
     [SkippableFact]
     public void Downsampler_ShouldSample_Works()
     {