Server: cut request-path lock contention and allocation; add load harness

Directory.Packages.props

@@ -14,6 +14,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />
+    <PackageVersion Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.1.21" />
     <PackageVersion Include="BitFaster.Caching" Version="2.6.1" />
     <PackageVersion Include="BouncyCastle.Cryptography" Version="2.7.0" />
     <PackageVersion Include="Confluent.Kafka" Version="2.15.1" />

docs/MigrationGuide.md

@@ -33,6 +33,7 @@ covers cross-cutting changes.
 - [Transport resource limits](#transport-resource-limits)
 - [Migrating channel subclasses that override HandleIncomingMessage](#migrating-channel-subclasses-that-override-handleincomingmessage)
 - [Migrating custom IUserDatabase implementations](#migrating-custom-iuserdatabase-implementations)
+- [Reading certificates from SecureChannelContext](#reading-certificates-from-securechannelcontext)
 - [Write service Value semantics follow OPC 10000-3 and OPC 10000-4](#write-service-value-semantics-follow-opc-10000-3-and-opc-10000-4)
 - [Address space permissions, Call, NodeManagement and locales follow OPC 10000-3](#address-space-permissions-call-nodemanagement-and-locales-follow-opc-10000-3)
 - [Migrating from 1.05.377 to 1.05.378](#migrating-from-105377-to-105378)
@@ -908,6 +909,31 @@ and no longer keeps metadata only in memory, so a store that cannot persist
 metadata should reject the write by returning `false` rather than silently
 accepting it.
 
+## Reading certificates from SecureChannelContext
+
+`SecureChannelContext.ClientChannelCertificate`, `ServerChannelCertificate` and
+`ChannelThumbprint` are `ByteString` instead of `byte[]`. The transport now shares
+one buffer of a channel's certificates with every request on that channel instead
+of copying the DER encoding per request, and a `ByteString` is a read-only view of
+it, so a request handler or an `IResourceIsolationClassifier` cannot alter the
+evidence later requests are checked against.
+
+The constructor still takes `byte[]` and wraps it without copying. A missing
+value is `IsNull`; an empty array stays empty, not null.
+
+| Before | After |
+|---|---|
+| `context.ClientChannelCertificate == null` | `context.ClientChannelCertificate.IsNull` |
+| `context.ClientChannelCertificate?.Length > 0` | `context.ClientChannelCertificate.Length > 0` |
+| `context.ClientChannelCertificate.AsSpan()` | `context.ClientChannelCertificate.Span` |
+| `Certificate.FromRawData(context.ClientChannelCertificate)` | `Certificate.FromRawData(context.ClientChannelCertificate.Memory)` |
+| `context.ClientChannelCertificate.ToByteString()` | `context.ClientChannelCertificate` |
+| passing the array to an API that takes `byte[]?` | `value.IsNull ? null : value.ToArray()` |
+
+Keep a missing certificate null when converting: the session signature helpers
+hash an empty array but skip a null one, so substituting `ToArray()` for a null
+value changes the signature data.
+
 ## ContentFilter NULL semantics follow OPC 10000-4 1.05.07
 
 `FilterEvaluator` applies the NULL rules of

docs/ServerScalability.md

@@ -9,6 +9,7 @@ This document describes how the reference server scales to large numbers of conc
 - [Admission control and rate limiting](#admission-control-and-rate-limiting)
 - [Held Publishes and the request-thread budget](#held-publishes-and-the-request-thread-budget)
 - [Session diagnostics cost](#session-diagnostics-cost)
+- [Garbage collector mode](#garbage-collector-mode)
 - [Configuration](#configuration)
 - [Scaling out](#scaling-out)
 - [See also](#see-also)
@@ -67,6 +68,20 @@ starving other requests.
 
 Creating a session or subscription registers a diagnostics node and marks the live `SessionDiagnostics` and `SubscriptionDiagnostics` arrays for refresh. The arrays are rebuilt on demand when they are read or monitored, and the rebuild is throttled so that a burst of session creates does not trigger a rebuild per create. Servers that do not need the live session-diagnostics arrays can turn them off with the `DiagnosticsEnabled` server setting to remove the cost entirely.
 
+## Garbage collector mode
+
+A server allocates per request on many threads at once. With the default workstation garbage collector the heap's gen0 budget is a few megabytes, so a loaded server collects hundreds of times per second and every collection suspends all request threads. In a 20-session Read load (100 values per request, 24-core machine) the workstation collector paused the process for about half of the wall-clock time; enabling Server GC on the same build doubled the request rate (21k to 44k Reads per second) and cut the 90th-percentile latency from 2.1 ms to 0.6 ms, at the price of a larger working set (about 430 MB instead of 210 MB).
+
+Enable Server GC in the server's project file:
+
+```xml
+<PropertyGroup>
+  <ServerGarbageCollection>true</ServerGarbageCollection>
+</PropertyGroup>
+```
+
+or set `DOTNET_gcServer=1` in the environment. On .NET 8 and later, Server GC adapts its heap count to the load (DATAS), so idle servers stay small. The console reference server enables it.
+
 ## Configuration
 
 The following settings size a single node for its hardware (see also the sizing notes in [Benchmarks.md](Benchmarks.md#server-session-scalability) and the admission-control settings in [RateLimiting.md](RateLimiting.md)):

perf/.gitignore

@@ -0,0 +1,4 @@
+traces/
+*.nettrace
+*.etlx
+ab-results.txt

perf/README.md

@@ -0,0 +1,76 @@
+# Server performance tooling
+
+Tools for measuring server latency, throughput and GC cost under load. They are not part of
+the product build (not in `UA.slnx`) and target `net10.0` only.
+
+| Path | Purpose |
+|---|---|
+| `ServerLoadHarness/` | Starts a reference server in a child process and drives load against it from the parent process. |
+| `TraceAnalyzer/` | Aggregates the `dotnet-trace` files the harness records (CPU, allocations, lock contention). |
+| `ab.sh`, `ab-summary.ps1` | Paired A/B runs of two harness builds, and the median summary. |
+
+The server runs in its own process, so the server-side numbers (CPU, allocated bytes, GC count
+and pause time, `Monitor.LockContentionCount`) exclude the load generator. The NUnit
+`LoadTest` fixture in `tests/Opc.Ua.Sessions.Tests` checks correctness under load; this harness
+measures it.
+
+## Build
+
+```bash
+CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release
+dotnet build perf/TraceAnalyzer -c Release
+dotnet tool install -g dotnet-trace
+```
+
+## Run
+
+```bash
+perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe run --scenario read --nodes 100 --sessions 20 --duration 15 --warmup 8
+```
+
+| Option | Default | Meaning |
+|---|---|---|
+| `--scenario` | `read` | `read`, `write`, `browse` (Server object) or `sub` (data-change subscriptions plus a writer). |
+| `--sessions` | 20 | Concurrent sessions. |
+| `--nodes` | 100 | Values per Read/Write request, or monitored items per subscription. |
+| `--inflight` | 1 | Outstanding requests per session (closed loop). |
+| `--security` | `none` | `none`, `sign` or `encrypt` (Basic256Sha256). |
+| `--subs`, `--pub`, `--write-interval` | 5, 100, 100 | Subscriptions per session, publishing interval and writer interval in ms (`sub`). |
+| `--duration`, `--warmup` | 20, 10 | Measurement window and warm-up in seconds. |
+| `--diag`, `--audit` | 0 | Server diagnostics and auditing. |
+| `--trace <file>` | | Record the measurement window with `dotnet-trace`. |
+| `--trace-profile` | `cpu` | `cpu` (sampled thread time), `alloc` (allocation ticks), `contention` or `gc`. |
+| `--out <file>` | | Append the result block to a file. |
+
+Environment variables prefixed `SERVER_` are passed to the server process without the prefix,
+for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes.
+
+The result block reports client throughput and latency percentiles, the server deltas for the
+window, and the server's allocated bytes and CPU time per request (or per notification).
+Allocated bytes per operation are deterministic and the most reliable signal. On a shared
+machine, throughput and latency vary from run to run, so compare builds with `ab.sh` rather
+than with two separate runs.
+
+## Analyse a trace
+
+```bash
+dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll alloc trace.nettrace 40
+dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll cpu trace.nettrace 40 Monitor.Enter_Slowpath
+dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll contention trace.nettrace 15
+```
+
+The sampled thread-time profile also samples blocked threads. The `cpu` mode drops samples
+whose leaf frame is a known wait (thread-pool semaphore, I/O completion poll, GC suspension),
+so the remaining samples approximate on-CPU time. Spin-waiting in `Monitor.Enter_Slowpath` or
+`Lock.TryEnterSlow` is real CPU and shows up there.
+
+## A/B comparison
+
+Build the baseline harness in a second worktree, then alternate the two builds:
+
+```bash
+git worktree add --detach C:/uabase origin/master
+cp -r perf C:/uabase/ && (cd C:/uabase && CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release)
+cd perf && BASELINE=C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe ./ab.sh
+pwsh -File ab-summary.ps1
+```

perf/ServerLoadHarness/CounterServer.cs

@@ -0,0 +1,162 @@
+/* ========================================================================
+ * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved.
+ *
+ * OPC Foundation MIT License 1.00
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The complete license agreement can be found here:
+ * http://opcfoundation.org/License/MIT/1.00/
+ * ======================================================================*/
+
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Threading;
+using Opc.Ua.Server;
+
+namespace Opc.Ua.Perf.ServerLoadHarness
+{
+    /// <summary>
+    /// Counter variables advanced by their read callback, so only sampling drives them.
+    /// Mirrors the subscription capacity workload of the o6-automation opcua-benchmarks
+    /// (UInt64 counter per monitored item, sampling groups, read-callback increments).
+    /// </summary>
+    internal sealed class CounterNodeManager : CustomNodeManager2
+    {
+        public const string Namespace = "urn:opcfoundation.org:perf:counters";
+
+        public CounterNodeManager(IServerInternal server, ApplicationConfiguration configuration, int count)
+            : base(server, configuration, useSamplingGroups: true, Namespace)
+        {
+            m_values = new ulong[count];
+            m_marked = new ulong[count];
+        }
+
+        public override void CreateAddressSpace(IDictionary<NodeId, IList<IReference>> externalReferences)
+        {
+            if (!externalReferences.TryGetValue(ObjectIds.ObjectsFolder, out IList<IReference>? references))
+            {
+                externalReferences[ObjectIds.ObjectsFolder] = references = new List<IReference>();
+            }
+            for (int i = 0; i < m_values.Length; i++)
+            {
+                int index = i;
+                var node = new BaseDataVariableState(null)
+                {
+                    NodeId = new NodeId((uint)i + 1, NamespaceIndex),
+                    BrowseName = new QualifiedName("Counter" + i.ToString(CultureInfo.InvariantCulture), NamespaceIndex),
+                    DisplayName = new LocalizedText("Counter" + i.ToString(CultureInfo.InvariantCulture)),
+                    TypeDefinitionId = VariableTypeIds.BaseDataVariableType,
+                    ReferenceTypeId = ReferenceTypeIds.Organizes,
+                    DataType = DataTypeIds.UInt64,
+                    ValueRank = ValueRanks.Scalar,
+                    MinimumSamplingInterval = 1,
+                    AccessLevel = AccessLevels.CurrentRead,
+                    UserAccessLevel = AccessLevels.CurrentRead,
+                    Value = new Variant((ulong)0),
+                    StatusCode = StatusCodes.Good,
+                    Timestamp = DateTimeUtc.Now
+                };
+                node.OnReadValue = (ISystemContext context, NodeState state, NumericRange range,
+                    QualifiedName encoding, ref Variant value, ref StatusCode status, ref DateTimeUtc timestamp) =>
+                {
+                    value = new Variant(Interlocked.Increment(ref m_values[index]));
+                    status = StatusCodes.Good;
+                    timestamp = DateTimeUtc.Now;
+                    return ServiceResult.Good;
+                };
+                node.AddReference(ReferenceTypeIds.Organizes, true, ObjectIds.ObjectsFolder);
+                references.Add(new NodeStateReference(ReferenceTypeIds.Organizes, false, node.NodeId));
+                AddPredefinedNode(SystemContext, node);
+            }
+        }
+
+        /// <summary>
+        /// Starts a measurement window.
+        /// </summary>
+        public void Mark()
+        {
+            for (int i = 0; i < m_values.Length; i++)
+            {
+                m_marked[i] = Interlocked.Read(ref m_values[i]);
+            }
+            m_markTimestamp = Stopwatch.GetTimestamp();
+        }
+
+        /// <summary>
+        /// Reports the sampling progress of the window: completed read callbacks per item
+        /// against the number the sampling interval calls for.
+        /// </summary>
+        public string Report(double samplingIntervalMs)
+        {
+            double elapsedMs = Stopwatch.GetElapsedTime(m_markTimestamp).TotalMilliseconds;
+            double expected = elapsedMs / samplingIntervalMs;
+            double min = double.MaxValue;
+            double sum = 0;
+            int belowTarget = 0;
+            for (int i = 0; i < m_values.Length; i++)
+            {
+                double progress = (Interlocked.Read(ref m_values[i]) - m_marked[i]) / expected;
+                min = Math.Min(min, progress);
+                sum += progress;
+                if (progress < 0.99)
+                {
+                    belowTarget++;
+                }
+            }
+            return FormattableString.Invariant(
+                $"counters={m_values.Length} samplesExpected={expected:F0} progressMin={min:P2} ") +
+                FormattableString.Invariant(
+                $"progressMean={sum / m_values.Length:P2} itemsBelow99={belowTarget}");
+        }
+
+        private readonly ulong[] m_values;
+        private readonly ulong[] m_marked;
+        private long m_markTimestamp = Stopwatch.GetTimestamp();
+    }
+
+    /// <summary>
+    /// A server exposing only the counter node manager in addition to the standard nodes.
+    /// </summary>
+    internal sealed class CounterServer : StandardServer
+    {
+        public CounterServer(ITelemetryContext telemetry, int count)
+            : base(telemetry)
+        {
+            m_count = count;
+        }
+
+        public CounterNodeManager? Counters { get; private set; }
+
+        protected override IMasterNodeManager CreateMasterNodeManager(
+            IServerInternal server,
+            ApplicationConfiguration configuration)
+        {
+            Counters = new CounterNodeManager(server, configuration, m_count);
+            return new MasterNodeManager(server, configuration, null, Counters);
+        }
+
+        private readonly int m_count;
+    }
+}