From a858f843896e09ee5dbaaa160b115b4ed2eb3430 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 00:11:56 +0200 Subject: [PATCH 01/16] Server: cut per-request locking and allocation on the request path Profiling a 20-session load (read/write/browse/subscribe) against the reference server showed the request path dominated by lock spinning and per-node allocation. Paired A/B medians (workstation GC, 24 cores): Read 1 value +45% req/s (CPU/req -37%, alloc/req -39%, lock contention -94%), Read 100 values +40%, encrypted Read 100 +48%, Browse +33%, Write 100 +13%, data-change notifications -17% alloc. - ObjectPool: lock-free slot array; ConcurrentBag.Count took every per-thread lock on each Return of the pooled response source. - FairRequestQueue: drop redundant gate entries (fail-fast pre-checks, stopped check, owner count folded into selection) and coalesce wake pulses so the bounded signal channel's lock is only taken when no token is pending. - Isolation provider: compare peer addresses without allocating; build the unsecured verification context only when a classifier is set. - TCP listener: cache the certificate DER copies per channel instead of copying X509 RawData for every request. - EndpointBase: resolve the request ActivitySource once instead of walking the stack (GetCallingAssembly) per request. - Node managers: reuse one ServerSystemContext copy per request in the per-node callbacks, remove a discarded copy, use plain dictionaries for per-request operation caches, build the validation cache in one pass and fix zero-capacity error lists. - ResourceManager: lock-free translation lookups from an immutable snapshot rebuilt after writes (Browse serialised on it). - MonitoredItem: share the default DataChangeFilter instead of allocating one per sampled value. - Reference server: enable Server GC; documented in ServerScalability.md (doubles Read throughput under load vs workstation GC). Co-Authored-By: Claude Opus 5.5 --- docs/ServerScalability.md | 15 +++ .../ConsoleReferenceServer.csproj | 5 + .../EndpointBase.EndpointIncomingRequest.cs | 3 +- src/Opc.Ua.Core/Stack/Server/EndpointBase.cs | 21 ++++ .../Stack/Server/FairRequestQueue.cs | 73 +++++++----- .../Stack/Tcp/TcpTransportListener.cs | 4 +- .../Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs | 35 ++++++ .../Stack/Tcp/UaSCBinaryChannel.cs | 2 +- .../Types/Utils/ValueTask/ObjectPool.cs | 54 +++++++-- .../NodeManager/AsyncCustomNodeManager.cs | 59 ++++------ .../NodeManager/CustomNodeManager.cs | 16 +-- .../NodeManager/CustomNodeManagerAsync.cs | 2 +- .../NodeManagerServiceDispatcher.cs | 17 +-- .../NodeManager/ResourceManager.cs | 111 ++++++++++++------ .../DefaultServerResourceIsolationProvider.cs | 58 ++++++--- src/Opc.Ua.Server/Server/OperationContext.cs | 40 +++++++ .../MonitoredItem/MonitoredItem.cs | 18 ++- 17 files changed, 374 insertions(+), 159 deletions(-) diff --git a/docs/ServerScalability.md b/docs/ServerScalability.md index b4b4f59236..53dc4c6095 100644 --- a/docs/ServerScalability.md +++ b/docs/ServerScalability.md @@ -9,6 +9,7 @@ This document describes how the reference server scales to large numbers of conc - [Admission control and rate limiting](#admission-control-and-rate-limiting) - [Held Publishes and the request-thread budget](#held-publishes-and-the-request-thread-budget) - [Session diagnostics cost](#session-diagnostics-cost) +- [Garbage collector mode](#garbage-collector-mode) - [Configuration](#configuration) - [Scaling out](#scaling-out) - [See also](#see-also) @@ -67,6 +68,20 @@ starving other requests. Creating a session or subscription registers a diagnostics node and marks the live `SessionDiagnostics` and `SubscriptionDiagnostics` arrays for refresh. The arrays are rebuilt on demand when they are read or monitored, and the rebuild is throttled so that a burst of session creates does not trigger a rebuild per create. Servers that do not need the live session-diagnostics arrays can turn them off with the `DiagnosticsEnabled` server setting to remove the cost entirely. +## Garbage collector mode + +A server allocates per request on many threads at once. With the default workstation garbage collector the heap's gen0 budget is a few megabytes, so a loaded server collects hundreds of times per second and every collection suspends all request threads. In a 20-session Read load (100 values per request, 24-core machine) the workstation collector paused the process for about half of the wall-clock time; enabling Server GC on the same build doubled the request rate (21k to 44k Reads per second) and cut the 90th-percentile latency from 2.1 ms to 0.6 ms, at the price of a larger working set (about 430 MB instead of 210 MB). + +Enable Server GC in the server's project file: + +```xml + + true + +``` + +or set `DOTNET_gcServer=1` in the environment. On .NET 8 and later, Server GC adapts its heap count to the load (DATAS), so idle servers stay small. The console reference server enables it. + ## Configuration The following settings size a single node for its hardware (see also the sizing notes in [Benchmarks.md](Benchmarks.md#server-session-scalability) and the admission-control settings in [RateLimiting.md](RateLimiting.md)): diff --git a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj index 5a0bb76678..1f36086530 100644 --- a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj +++ b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj @@ -11,6 +11,11 @@ Quickstarts 46345736-a30f-4466-b3bb-42548ecfaacc true + + true diff --git a/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs b/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs index 86d308794f..e20995acc1 100644 --- a/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs +++ b/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs @@ -109,8 +109,7 @@ public readonly async ValueTask CallAsync(CancellationToken cancellationToken = try { Activity? activity = null; - ActivitySource activitySource = m_endpoint.MessageContext.Telemetry - .GetActivitySource(); + ActivitySource activitySource = m_endpoint.RequestActivitySource; if (activitySource.HasListeners()) { // extract trace information from the request header if available diff --git a/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs b/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs index 40ecf596b3..3c479e9dca 100644 --- a/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs +++ b/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs @@ -518,6 +518,26 @@ internal static ServiceFault CreateFault( /// The message context. protected IServiceMessageContext MessageContext => m_server!.MessageContext; + /// + /// The activity source for incoming requests. Resolved once per telemetry + /// context: the GetActivitySource extension walks the stack to find the + /// calling assembly, which is too expensive to repeat for every request. + /// + private ActivitySource RequestActivitySource + { + get + { + ITelemetryContext telemetry = MessageContext.Telemetry; + Tuple? cached = m_activitySource; + if (cached == null || !ReferenceEquals(cached.Item1, telemetry)) + { + cached = Tuple.Create(telemetry, telemetry.GetActivitySource()); + m_activitySource = cached; + } + return cached.Item2; + } + } + /// /// Returns the description for the endpoint /// @@ -584,6 +604,7 @@ protected delegate ValueTask InvokeService( private IServiceHostBase? m_host; private IServerBase? m_server; + private Tuple? m_activitySource; } /// diff --git a/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs b/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs index 056593e9b6..a735900d5f 100644 --- a/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs +++ b/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs @@ -78,16 +78,11 @@ public FairRequestQueue( /// /// Gets the number of requests still waiting for execution admission. /// - internal int Count - { - get - { - lock (m_gate) - { - return m_count; - } - } - } + /// + /// Read without the queue gate: the count is only mutated under the gate and + /// callers use it as a snapshot (worker growth heuristics, diagnostics). + /// + internal int Count => Volatile.Read(ref m_count); /// /// Gets the number of scheduling FIFOs with queued requests: one per owner in weighted @@ -128,18 +123,17 @@ public bool TryEnqueue( error = StatusCodes.BadRequestCancelledByClient; return false; } - lock (m_gate) + // Fail fast without the gate; both conditions are re-checked under the + // gate before the entry is published. + if (Volatile.Read(ref m_stopped)) { - if (m_stopped) - { - error = StatusCodes.BadServerHalted; - return false; - } - if (m_count >= m_maxQueuedRequests) - { - error = StatusCodes.BadServerTooBusy; - return false; - } + error = StatusCodes.BadServerHalted; + return false; + } + if (Volatile.Read(ref m_count) >= m_maxQueuedRequests) + { + error = StatusCodes.BadServerTooBusy; + return false; } ResourceIsolationOwner owner = Classify(request); @@ -225,8 +219,8 @@ public bool TryEnqueue( /// public bool TryDequeue([NotNullWhen(true)] out Entry? result) { - int attempts = OwnerCount; - while (attempts-- > 0 && TrySelect(out Entry? candidate, out long version)) + int attempts = -1; + while (TrySelect(ref attempts, out Entry? candidate, out long version)) { IDisposable? executionLease = null; StatusCode error = StatusCodes.Good; @@ -323,12 +317,9 @@ public async ValueTask DequeueAsync(CancellationToken cancellationToken) while (true) { cancellationToken.ThrowIfCancellationRequested(); - lock (m_gate) + if (Volatile.Read(ref m_stopped)) { - if (m_stopped) - { - throw new OperationCanceledException(cancellationToken); - } + throw new OperationCanceledException(cancellationToken); } if (TryDequeue(out Entry? entry)) { @@ -337,6 +328,10 @@ public async ValueTask DequeueAsync(CancellationToken cancellationToken) try { await m_signals.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + // The token is consumed; let the next Pulse publish a new one. This + // reader dequeues after the reset, so a Pulse skipped before it is + // still observed. + Volatile.Write(ref m_signalPending, 0); } catch (ChannelClosedException) { @@ -434,13 +429,19 @@ private static bool IsControlRequest(IServiceRequest request) /// /// Selects one eligible owner head while excluding concurrent provider admission attempts. + /// The first call of a dequeue pass ( negative) snapshots the + /// number of scheduling FIFOs under the same gate acquisition; each call consumes one attempt. /// - private bool TrySelect([NotNullWhen(true)] out Entry? entry, out long version) + private bool TrySelect(ref int attempts, [NotNullWhen(true)] out Entry? entry, out long version) { lock (m_gate) { version = Volatile.Read(ref m_version); - if (!m_stopped && !m_selecting) + if (attempts < 0) + { + attempts = m_owners.Count; + } + if (attempts-- > 0 && !m_stopped && !m_selecting) { int remaining = m_round.Count; while (remaining-- > 0 && m_round.First != null) @@ -539,9 +540,16 @@ private void Changed() /// /// Retains at most one wake token and hands it to only one reader, never broadcasting. /// + /// + /// Pulse runs on every admission, dispatch and capacity release. While a token is + /// already pending the call returns without entering the signal channel's lock, + /// which otherwise serialises every request thread on it. + /// private void Pulse() { if (Volatile.Read(ref m_count) > 0 && !Volatile.Read(ref m_stopped) && + Volatile.Read(ref m_signalPending) == 0 && + Interlocked.Exchange(ref m_signalPending, 1) == 0 && m_signals.Writer.TryWrite(true)) { Interlocked.Increment(ref m_wakeSignalCount); @@ -857,6 +865,11 @@ public void Dispose() /// private readonly Channel m_signals = Channel.CreateBounded(1); + /// + /// 1 while a wake token is in and not yet consumed. + /// + private int m_signalPending; + /// /// Number of linked requests. /// diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs index a4c3823e4c..b12a02cd54 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs @@ -1721,8 +1721,8 @@ private async void OnRequestReceivedAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs index 89905d7c23..e0ff3022f8 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs @@ -99,6 +99,20 @@ protected string SecurityPolicyUri /// internal Certificate? ClientCertificate { get; set; } + /// + /// The DER encoding of , copied once per certificate. + /// + /// + /// Certificate.RawData returns a new copy on every access; the listener needs the + /// bytes for every request it dispatches. + /// + internal byte[]? ClientCertificateRawData => GetCachedRawData(ClientCertificate, ref m_clientCertificateRawData); + + /// + /// The DER encoding of , copied once per certificate. + /// + internal byte[]? ServerCertificateRawData => GetCachedRawData(ServerCertificate, ref m_serverCertificateRawData); + /// /// The client certificate chain. /// @@ -2459,5 +2473,26 @@ private static RsaUtils.Padding GetAsymmetricPadding(SecurityPolicyInfo policy) private bool m_noncesDisposed; private Nonce? m_localNonce; private Nonce? m_remoteNonce; + private Tuple? m_clientCertificateRawData; + private Tuple? m_serverCertificateRawData; + + /// + /// Returns the cached DER copy of a certificate, refreshing it when the + /// certificate instance changes (renewal, reassignment). + /// + private static byte[]? GetCachedRawData(Certificate? certificate, ref Tuple? cache) + { + if (certificate == null) + { + return null; + } + Tuple? cached = Volatile.Read(ref cache); + if (cached == null || !ReferenceEquals(cached.Item1, certificate)) + { + cached = Tuple.Create(certificate, certificate.RawData); + Volatile.Write(ref cache, cached); + } + return cached.Item2; + } } } diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs index b8dd806544..adf4af4461 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs @@ -646,7 +646,7 @@ private bool SaveChunk( { var context = new SecureChannelContext( GlobalChannelId, EndpointDescription, RequestEncoding.Binary, - ClientCertificate?.RawData, ServerCertificate?.RawData, ChannelThumbprint, + ClientCertificateRawData, ServerCertificateRawData, ChannelThumbprint, (Transport?.RemoteEndpoint as System.Net.IPEndPoint)?.Address); owner = isolation is IResourceIsolationReassemblyProvider reassembly ? reassembly.ClassifyReassembly(context) : isolation.Classify(context); diff --git a/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs b/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs index 48bc0c53e7..2b0eee3480 100644 --- a/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs +++ b/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs @@ -27,31 +27,45 @@ * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ + using System; -using System.Collections.Concurrent; +using System.Threading; namespace Opc.Ua { /// - /// A simple object pool implementation. + /// A simple bounded, lock-free object pool. /// + /// + /// Get and Return run once per service request on the server hot path and + /// typically on different threads (the request is taken on a receive thread + /// and returned on whichever thread awaits the response). A fixed slot array + /// with interlocked exchange keeps both operations wait-free; a + /// ConcurrentBag would steal across thread-local lists and its Count takes + /// every per-thread lock. + /// /// The type of object to pool. internal class ObjectPool where T : class { - private readonly ConcurrentBag m_objects; private readonly Func m_objectGenerator; - private readonly int m_maxSize; + private readonly T?[] m_items; + private T? m_firstItem; /// /// Initializes a new instance of the class. /// /// The function to generate new objects. - /// The maximum size of the pool. + /// The maximum number of retained objects (capped at four per processor). public ObjectPool(Func objectGenerator, int maxSize) { m_objectGenerator = objectGenerator ?? throw new ArgumentNullException(nameof(objectGenerator)); - m_maxSize = maxSize > 0 ? maxSize : throw new ArgumentOutOfRangeException(nameof(maxSize)); - m_objects = []; + if (maxSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxSize)); + } + // Retained objects only need to cover the concurrently outstanding ones; + // more slots would only lengthen the scan when the pool runs dry. + m_items = new T?[Math.Max(Math.Min(maxSize, Environment.ProcessorCount * 4), 1) - 1]; } /// @@ -60,11 +74,22 @@ public ObjectPool(Func objectGenerator, int maxSize) /// An object from the pool or a new one if the pool is empty. public T Get() { - if (m_objects.TryTake(out T? item)) + T? item = m_firstItem; + if (item != null && Interlocked.CompareExchange(ref m_firstItem, null, item) == item) { return item; } + T?[] items = m_items; + for (int i = 0; i < items.Length; i++) + { + item = items[i]; + if (item != null && Interlocked.CompareExchange(ref items[i], null, item) == item) + { + return item; + } + } + return m_objectGenerator(); } @@ -74,9 +99,18 @@ public T Get() /// The object to return. public void Return(T item) { - if (m_objects.Count < m_maxSize) + if (m_firstItem == null && Interlocked.CompareExchange(ref m_firstItem, item, null) == null) + { + return; + } + + T?[] items = m_items; + for (int i = 0; i < items.Length; i++) { - m_objects.Add(item); + if (Interlocked.CompareExchange(ref items[i], item, null) == null) + { + return; + } } } } diff --git a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs index 8ef00085e0..88cb9791b8 100644 --- a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs @@ -643,6 +643,15 @@ ILocalAddressSpace ILocalAddressSpaceSource.CreateLocalAddressSpace() /// public ServerSystemContext SystemContext { get; } + /// + /// Returns the system context for an operation, reusing the copy already made + /// for the same request. Use it in callbacks that run once per node of a request. + /// + private ServerSystemContext GetOperationSystemContext(OperationContext context) + { + return context != null ? context.GetSystemContext(SystemContext) : SystemContext.Copy(context!); + } + /// /// Gets the default index for the node manager's namespace. /// @@ -4345,7 +4354,7 @@ public async ValueTask GetNodeMetadataAsync( BrowseResultMask resultMask, CancellationToken cancellationToken = default) { - ServerSystemContext systemContext = SystemContext.Copy(context); + ServerSystemContext systemContext = GetOperationSystemContext(context); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(targetHandle); @@ -4716,8 +4725,6 @@ protected async ValueTask GetReferenceDescriptionAsync( ContinuationPoint continuationPoint, CancellationToken cancellationToken = default) { - _ = SystemContext.Copy(context); - // create the type definition reference. var description = new ReferenceDescription { NodeId = reference.TargetId }; description.SetReferenceType( @@ -4821,7 +4828,7 @@ public virtual async ValueTask TranslateBrowsePathAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(sourceHandle); @@ -4944,7 +4951,7 @@ public virtual async ValueTask ReadAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); for (int ii = 0; ii < nodesToRead.Count; ii++) @@ -5208,7 +5215,7 @@ public virtual async ValueTask WriteAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); @@ -5936,7 +5943,7 @@ public virtual async ValueTask HistoryReadAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); for (int ii = 0; ii < nodesToRead.Count; ii++) @@ -6808,7 +6815,7 @@ public virtual async ValueTask HistoryUpdateAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); @@ -7422,7 +7429,7 @@ public virtual async ValueTask FindMethodStateAsync( } ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); NodeHandle? handle = await GetManagerHandleAsync( systemContext, @@ -7482,7 +7489,7 @@ public virtual async ValueTask CallAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); for (int ii = 0; ii < methodsToCall.Count; ii++) { @@ -8244,7 +8251,7 @@ public virtual async ValueTask RestoreMonitoredItemsAsync( } ServerSystemContext systemContext = SystemContext.Copy(); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var restoredItems = new List(); @@ -8416,7 +8423,7 @@ public virtual async ValueTask CreateMonitoredItemsAsync( { using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var createdItems = new List(); @@ -10369,7 +10376,7 @@ public virtual ValueTask IsNodeInViewAsync( if (handle.Node != null) { return new ValueTask( - IsNodeInView(SystemContext.Copy(context), viewId, handle.Node)); + IsNodeInView(GetOperationSystemContext(context), viewId, handle.Node)); } return new ValueTask(false); @@ -10390,7 +10397,7 @@ public virtual ValueTask IsNodeInViewAsync( bool permissionsOnly, CancellationToken cancellationToken = default) { - ServerSystemContext systemContext = SystemContext.Copy(context); + ServerSystemContext systemContext = GetOperationSystemContext(context); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(targetHandle); @@ -10408,8 +10415,6 @@ public virtual ValueTask IsNodeInViewAsync( return null!; } - var values = new Variant[3]; - // construct the meta-data object. var metadata = new NodeMetadata(target, target.NodeId) { @@ -10420,25 +10425,10 @@ public virtual ValueTask IsNodeInViewAsync( if (uniqueNodesServiceAttributesCache != null) { NodeId key = handle.NodeId; - if (uniqueNodesServiceAttributesCache.ContainsKey(key)) - { - if (uniqueNodesServiceAttributesCache[key].Length != 3) - { - ReadAndCacheValidationAttributes( - uniqueNodesServiceAttributesCache, - systemContext, - target, - key, - ref values); - } - else - { - // Retrieve value from cache - values = uniqueNodesServiceAttributesCache[key]; - } - } - else + if (!uniqueNodesServiceAttributesCache.TryGetValue(key, out Variant[]? values) || + values.Length != 3) { + values = new Variant[3]; ReadAndCacheValidationAttributes( uniqueNodesServiceAttributesCache, systemContext, @@ -10451,6 +10441,7 @@ public virtual ValueTask IsNodeInViewAsync( } // All other calls that do not use the cache else if (permissionsOnly) { + var values = new Variant[3]; ReadValidationAttributes(systemContext, target, ref values); SetAccessAndRolePermissions(values, metadata); } diff --git a/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs index 98792081e1..b0a8cb599d 100644 --- a/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs @@ -2375,7 +2375,7 @@ public virtual void TranslateBrowsePath( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(sourceHandle); @@ -2486,7 +2486,7 @@ public virtual void Read( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); lock (Lock) @@ -2741,7 +2741,7 @@ public virtual void Write( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var nodesToNotify = new List(); @@ -3391,7 +3391,7 @@ public virtual void HistoryRead( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); lock (Lock) @@ -3910,7 +3910,7 @@ public virtual void HistoryUpdate( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); lock (Lock) @@ -4420,7 +4420,7 @@ public virtual MethodState FindMethodState( } ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); NodeHandle? handle = GetManagerHandle( systemContext, @@ -4939,7 +4939,7 @@ public virtual void RestoreMonitoredItems( } ServerSystemContext systemContext = SystemContext.Copy(); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); for (int ii = 0; ii < itemsToRestore.Count; ii++) @@ -5098,7 +5098,7 @@ public virtual void CreateMonitoredItems( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var createdItems = new List(); diff --git a/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs b/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs index 1b19d5c4bf..2d68511c9b 100644 --- a/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs +++ b/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs @@ -103,7 +103,7 @@ private ServiceResult CallSynchronousMethod( ArrayOf methodsToCall, IList errors) { - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); for (int ii = 0; ii < methodsToCall.Count; ii++) { CallMethodRequest request = methodsToCall[ii]; diff --git a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs index 4ca7a05d40..251eeda8d1 100644 --- a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs +++ b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs @@ -714,7 +714,6 @@ private static void PrepareValidationCache( ArrayOf nodesList, out Dictionary uniqueNodesServiceAttributes) { - var uniqueNodes = new HashSet(); Type listType = typeof(T); if (listType != typeof(ReadValueId) && @@ -727,6 +726,8 @@ private static void PrepareValidationCache( nameof(nodesList)); } + // uniqueNodesServiceAttributes is the place where the attributes for each unique nodeId are kept on the services + uniqueNodesServiceAttributes = new Dictionary(nodesList.Count); for (int i = 0; i < nodesList.Count; i++) { NodeId nodeId = default; @@ -746,15 +747,9 @@ private static void PrepareValidationCache( if (!nodeId.IsNull) { - uniqueNodes.Add(nodeId); + uniqueNodesServiceAttributes.TryAdd(nodeId, []); } } - // uniqueNodesReadAttributes is the place where the attributes for each unique nodeId are kept on the services - uniqueNodesServiceAttributes = []; - foreach (NodeId uniqueNode in uniqueNodes) - { - uniqueNodesServiceAttributes.Add(uniqueNode, []); - } } /// @@ -1223,7 +1218,7 @@ private async ValueTask UpdateReferenceDescriptionAsync( var diagnosticInfos = new List(nodesToRead.Count); // create empty list of errors. - var errors = new List(values.Count); + var errors = new List(nodesToRead.Count); for (int ii = 0; ii < nodesToRead.Count; ii++) { errors.Add(null!); @@ -1370,7 +1365,7 @@ await nodeManager.ReadAsync( // pre-validate items. bool validItems = false; // create empty list of errors. - var errors = new List(results.Count); + var errors = new List(nodesToRead.Count); for (int ii = 0; ii < nodesToRead.Count; ii++) { errors.Add(null!); @@ -1709,7 +1704,7 @@ await nodeManager.WriteAsync( bool validItems = false; // create empty list of errors. - var errors = new List(results.Count); + var errors = new List(nodesToUpdate.Count); for (int ii = 0; ii < nodesToUpdate.Count; ii++) { errors.Add(null!); diff --git a/src/Opc.Ua.Server/NodeManager/ResourceManager.cs b/src/Opc.Ua.Server/NodeManager/ResourceManager.cs index 6b5a6e5834..57719f2729 100644 --- a/src/Opc.Ua.Server/NodeManager/ResourceManager.cs +++ b/src/Opc.Ua.Server/NodeManager/ResourceManager.cs @@ -247,6 +247,7 @@ public void Add(string key, string locale, string text) { TranslationTable? table = GetTable(culture.Name); table!.Translations[key] = text; + m_translationSnapshot = null; } } @@ -284,6 +285,7 @@ public void Add(string locale, IDictionary translations) { table!.Translations[translation.Key] = translation.Value; } + m_translationSnapshot = null; } } @@ -406,38 +408,34 @@ protected virtual LocalizedText Translate( : []; #endif // If only mul/qst is requested, return all available translations for the key. + TranslationTable[] tables = GetTranslationSnapshot(); if (preferredLocales.Count == 1) { - lock (m_lock) + foreach (TranslationTable table in tables) { - foreach (TranslationTable table in m_translationTables) + if (table.Translations + .TryGetValue((info.Key ?? info.Text)!, out string? translation)) { - if (table.Translations - .TryGetValue((info.Key ?? info.Text)!, out string? translation)) - { - translations[table!.Locale!.Name] = translation!; - } + translations[table!.Locale!.Name] = translation!; } } } else { // mul/qst + specific locales: return only those translations - lock (m_lock) + for (int i = 1; i < preferredLocales.Count; i++) { - for (int i = 1; i < preferredLocales.Count; i++) + string? translation = FindBestTranslation( + tables, + preferredLocales.Slice(i, 1), + (info.Key ?? info.Text)!, + out CultureInfo culture); + if (translation != null) { - string? translation = FindBestTranslation( - preferredLocales.Slice(i, 1), - (info.Key ?? info.Text)!, - out CultureInfo culture); - if (translation != null) - { - // label the text with the locale of the table it came - // from, which may be another region of the requested - // language. - translations[culture.Name] = translation; - } + // label the text with the locale of the table it came + // from, which may be another region of the requested + // language. + translations[culture.Name] = translation; } } } @@ -453,20 +451,18 @@ protected virtual LocalizedText Translate( string? translatedText = info.Text; CultureInfo culture = CultureInfo.InvariantCulture; - lock (m_lock) - { - translatedText = FindBestTranslation( - preferredLocales, - (info.Key ?? info.Text)!, - out culture); + translatedText = FindBestTranslation( + GetTranslationSnapshot(), + preferredLocales, + (info.Key ?? info.Text)!, + out culture); - // use the default if no translation available. - if (translatedText == null) - { - return preferredLocales.Count > 0 && defaultText.Translations == null && info.Text != null - ? new LocalizedText(info) - : defaultText.FilterByPreferredLocales(preferredLocales); - } + // use the default if no translation available. + if (translatedText == null) + { + return preferredLocales.Count > 0 && defaultText.Translations == null && info.Text != null + ? new LocalizedText(info) + : defaultText.FilterByPreferredLocales(preferredLocales); } // construct translated localized text. @@ -483,6 +479,41 @@ private class TranslationTable public SortedDictionary Translations = []; } + /// + /// Returns an immutable copy of the translation tables. Translations are looked up + /// for every localized text a service returns (each DisplayName of a Browse), while + /// the tables are written almost only at startup, so readers share a snapshot instead + /// of serialising on the lock. The dictionaries are copied, never shared with a writer. + /// + private TranslationTable[] GetTranslationSnapshot() + { + TranslationTable[]? snapshot = Volatile.Read(ref m_translationSnapshot); + if (snapshot != null) + { + return snapshot; + } + + lock (m_lock) + { + snapshot = m_translationSnapshot; + if (snapshot == null) + { + snapshot = new TranslationTable[m_translationTables.Count]; + for (int ii = 0; ii < snapshot.Length; ii++) + { + TranslationTable table = m_translationTables[ii]; + snapshot[ii] = new TranslationTable + { + Locale = table.Locale, + Translations = new SortedDictionary(table.Translations) + }; + } + Volatile.Write(ref m_translationSnapshot, snapshot); + } + return snapshot; + } + } + /// /// Finds the translation table for the locale. Creates a new table if it does not exist. /// @@ -504,6 +535,7 @@ private class TranslationTable // add table. var table = new TranslationTable { Locale = new CultureInfo(locale) }; m_translationTables.Add(table); + m_translationSnapshot = null; return table; } @@ -569,7 +601,8 @@ private static bool ContainsAllLocales( /// /// Finds the best translation for the requested locales. /// - private string? FindBestTranslation( + private static string? FindBestTranslation( + TranslationTable[] tables, ArrayOf preferredLocales, string key, out CultureInfo culture) @@ -602,9 +635,9 @@ private static bool ContainsAllLocales( // search for translation. string? translatedText = null; - for (int ii = 0; ii < m_translationTables.Count; ii++) + for (int ii = 0; ii < tables.Length; ii++) { - TranslationTable translationTable = m_translationTables[ii]; + TranslationTable translationTable = tables[ii]; // all done if exact match found (locale ids are case-insensitive, RFC 5646). if (string.Equals( @@ -706,6 +739,12 @@ private LocalizedText TranslateSymbolicId( private readonly Lock m_lock = new(); private readonly List m_translationTables; + + /// + /// Immutable copy of read without the lock; + /// null after a change until the next translation rebuilds it. + /// + private TranslationTable[]? m_translationSnapshot; private Dictionary? m_statusCodeMapping; private Dictionary? m_symbolicIdMapping; } diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index db3f6881e5..1055942831 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -273,22 +273,25 @@ public StatusCode GetRevalidationStatus( ? StatusCodes.BadSessionIdInvalid : StatusCodes.BadServerTooBusy; } - bool secure = original.Channel.SecurityMode is - MessageSecurityMode.Sign or MessageSecurityMode.SignAndEncrypt && - original.Channel.SecurityPolicyUri != SecurityPolicies.None; - SecureChannelContext verifiedContext = secure ? channelContext : new SecureChannelContext( - channelContext.SecureChannelId, channelContext.EndpointDescription, channelContext.MessageEncoding, - peerAddress: channelContext.PeerAddress) - { UpstreamIdentity = channelContext.UpstreamIdentity }; - if (m_classifier?.TryClassify(verifiedContext, current, out ResourceIsolationIdentity identity) == true || - m_classifier?.TryClassifyIngress( - channelContext.PeerAddress == null ? null : new IPEndPoint(channelContext.PeerAddress, 0), - out identity) == true) - { - string key = identity.Class == ResourceIsolationClass.Trusted - ? GetTrustedOwnerKey(identity.Key) : "mapped:" + identity.Key; - return identity.Class == owner.Class && key == owner.Key - ? StatusCodes.Good : StatusCodes.BadServerTooBusy; + if (m_classifier != null) + { + bool secure = original.Channel.SecurityMode is + MessageSecurityMode.Sign or MessageSecurityMode.SignAndEncrypt && + original.Channel.SecurityPolicyUri != SecurityPolicies.None; + SecureChannelContext verifiedContext = secure ? channelContext : new SecureChannelContext( + channelContext.SecureChannelId, channelContext.EndpointDescription, channelContext.MessageEncoding, + peerAddress: channelContext.PeerAddress) + { UpstreamIdentity = channelContext.UpstreamIdentity }; + if (m_classifier.TryClassify(verifiedContext, current, out ResourceIsolationIdentity identity) || + m_classifier.TryClassifyIngress( + channelContext.PeerAddress == null ? null : new IPEndPoint(channelContext.PeerAddress, 0), + out identity)) + { + string key = identity.Class == ResourceIsolationClass.Trusted + ? GetTrustedOwnerKey(identity.Key) : "mapped:" + identity.Key; + return identity.Class == owner.Class && key == owner.Key + ? StatusCodes.Good : StatusCodes.BadServerTooBusy; + } } ResourceIsolationClass expected = current != null && controlRequest ? ResourceIsolationClass.Control @@ -861,16 +864,35 @@ public ChannelSnapshot(SecureChannelContext context) /// public bool Matches(SecureChannelContext context) { - byte[]? peer = context.PeerAddress?.GetAddressBytes(); return m_channelId == context.SecureChannelId && SecurityPolicyUri == context.EndpointDescription?.SecurityPolicyUri && SecurityMode == (context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid) && (SecurityPolicyUri == SecurityPolicies.None || m_certificate.AsSpan().SequenceEqual(context.ClientChannelCertificate.AsSpan())) && - m_peer.AsSpan().SequenceEqual(peer.AsSpan()) && + PeerMatches(context.PeerAddress) && ReferenceEquals(m_upstreamIdentity, context.UpstreamIdentity); } + /// + /// Compares the captured peer address bytes; runs for every request, so the + /// current address is written to the stack rather than copied to a new array. + /// + private bool PeerMatches(IPAddress? peer) + { + if (peer == null) + { + return m_peer == null || m_peer.Length == 0; + } +#if NET + Span bytes = stackalloc byte[16]; + if (peer.TryWriteBytes(bytes, out int written)) + { + return m_peer.AsSpan().SequenceEqual(bytes[..written]); + } +#endif + return m_peer.AsSpan().SequenceEqual(peer.GetAddressBytes()); + } + private readonly string m_channelId; private readonly byte[]? m_certificate; private readonly byte[]? m_peer; diff --git a/src/Opc.Ua.Server/Server/OperationContext.cs b/src/Opc.Ua.Server/Server/OperationContext.cs index 1b1a44e046..d9bf203a62 100644 --- a/src/Opc.Ua.Server/Server/OperationContext.cs +++ b/src/Opc.Ua.Server/Server/OperationContext.cs @@ -395,8 +395,48 @@ internal void AttachSessionlessLease(IDisposable lease) m_sessionlessLease = lease; } + /// + /// Returns a copy of a node manager's system context bound to this operation, + /// reusing the copy made for the previous call with the same template. + /// + /// + /// Per-node node manager callbacks (permission and node metadata, reference + /// descriptions) otherwise clone the system context once for every node of a + /// request. The copies are equal for one request, so the last one is kept. + /// + /// The node manager's system context. + internal ServerSystemContext GetSystemContext(ServerSystemContext template) + { + SystemContextCacheEntry? entry = m_systemContextCache; + if (entry != null && ReferenceEquals(entry.Template, template)) + { + return entry.Copy; + } + + ServerSystemContext copy = template.Copy(this); + m_systemContextCache = new SystemContextCacheEntry(template, copy); + return copy; + } + + /// + /// A system context copy and the template it was made from, published as one + /// reference so concurrent readers never pair a copy with the wrong template. + /// + private sealed class SystemContextCacheEntry + { + public SystemContextCacheEntry(ServerSystemContext template, ServerSystemContext copy) + { + Template = template; + Copy = copy; + } + + public ServerSystemContext Template { get; } + public ServerSystemContext Copy { get; } + } + private IDisposable? m_requestScope; private IDisposable? m_sessionlessLease; + private SystemContextCacheEntry? m_systemContextCache; private static uint s_lastRequestId; } } diff --git a/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs b/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs index 4b9e0b1a6a..0d3264fb72 100644 --- a/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs +++ b/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs @@ -2341,7 +2341,6 @@ public static bool ValueChanged( } // select default data change filters. - const double deadband = 0.0; DeadbandType deadbandType = DeadbandType.None; DataChangeTrigger trigger = DataChangeTrigger.StatusValue; @@ -2361,11 +2360,8 @@ public static bool ValueChanged( } else { - filter = new DataChangeFilter - { - DeadbandType = (uint)deadbandType, - DeadbandValue = deadband - }; + // runs for every sampled value of an unfiltered item; AreEqual only reads it. + filter = s_defaultDataChangeFilter; } // get the current status. @@ -2713,6 +2709,16 @@ private void DisposeQueueHandlers() } private readonly Lock m_lock = new(); + + /// + /// The filter applied when a monitored item has none: no deadband, StatusValue trigger. + /// Shared and never modified. + /// + private static readonly DataChangeFilter s_defaultDataChangeFilter = new() + { + DeadbandType = (uint)DeadbandType.None, + DeadbandValue = 0.0 + }; private readonly ILogger m_logger; private bool m_isDisposed; private bool m_isAttaching; From e4867fcb5aa41e8d326eb8464246a45edc37e560 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 07:06:34 +0200 Subject: [PATCH 02/16] Add server load harness and trace analyzer under perf/ ServerLoadHarness runs a reference server in a child process and drives read/write/browse/subscription load against it, reporting client latency percentiles and the server's CPU, allocation, GC and lock-contention deltas, optionally recording the window with dotnet-trace. TraceAnalyzer aggregates those traces; ab.sh/ab-summary.ps1 run paired comparisons of two builds. Not part of UA.slnx. Co-Authored-By: Claude Opus 5.5 --- Directory.Packages.props | 1 + perf/.gitignore | 4 + perf/README.md | 76 ++ perf/ServerLoadHarness/Program.cs | 647 ++++++++++++++++++ .../ServerLoadHarness.csproj | 20 + perf/TraceAnalyzer/Program.cs | 174 +++++ perf/TraceAnalyzer/TraceAnalyzer.csproj | 17 + perf/ab-summary.ps1 | 25 + perf/ab.sh | 22 + 9 files changed, 986 insertions(+) create mode 100644 perf/.gitignore create mode 100644 perf/README.md create mode 100644 perf/ServerLoadHarness/Program.cs create mode 100644 perf/ServerLoadHarness/ServerLoadHarness.csproj create mode 100644 perf/TraceAnalyzer/Program.cs create mode 100644 perf/TraceAnalyzer/TraceAnalyzer.csproj create mode 100644 perf/ab-summary.ps1 create mode 100644 perf/ab.sh diff --git a/Directory.Packages.props b/Directory.Packages.props index 33df6f16f1..29a206953a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,6 +14,7 @@ + diff --git a/perf/.gitignore b/perf/.gitignore new file mode 100644 index 0000000000..ad11d0c47d --- /dev/null +++ b/perf/.gitignore @@ -0,0 +1,4 @@ +traces/ +*.nettrace +*.etlx +ab-results.txt diff --git a/perf/README.md b/perf/README.md new file mode 100644 index 0000000000..5c1a70ae36 --- /dev/null +++ b/perf/README.md @@ -0,0 +1,76 @@ +# Server performance tooling + +Tools for measuring server latency, throughput and GC cost under load. They are not part of +the product build (not in `UA.slnx`) and target `net10.0` only. + +| Path | Purpose | +|---|---| +| `ServerLoadHarness/` | Starts a reference server in a child process and drives load against it from the parent process. | +| `TraceAnalyzer/` | Aggregates the `dotnet-trace` files the harness records (CPU, allocations, lock contention). | +| `ab.sh`, `ab-summary.ps1` | Paired A/B runs of two harness builds, and the median summary. | + +The server runs in its own process, so the server-side numbers (CPU, allocated bytes, GC count +and pause time, `Monitor.LockContentionCount`) exclude the load generator. The NUnit +`LoadTest` fixture in `tests/Opc.Ua.Sessions.Tests` checks correctness under load; this harness +measures it. + +## Build + +```bash +CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release +dotnet build perf/TraceAnalyzer -c Release +dotnet tool install -g dotnet-trace +``` + +## Run + +```bash +perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe run --scenario read --nodes 100 --sessions 20 --duration 15 --warmup 8 +``` + +| Option | Default | Meaning | +|---|---|---| +| `--scenario` | `read` | `read`, `write`, `browse` (Server object) or `sub` (data-change subscriptions plus a writer). | +| `--sessions` | 20 | Concurrent sessions. | +| `--nodes` | 100 | Values per Read/Write request, or monitored items per subscription. | +| `--inflight` | 1 | Outstanding requests per session (closed loop). | +| `--security` | `none` | `none`, `sign` or `encrypt` (Basic256Sha256). | +| `--subs`, `--pub`, `--write-interval` | 5, 100, 100 | Subscriptions per session, publishing interval and writer interval in ms (`sub`). | +| `--duration`, `--warmup` | 20, 10 | Measurement window and warm-up in seconds. | +| `--diag`, `--audit` | 0 | Server diagnostics and auditing. | +| `--trace ` | | Record the measurement window with `dotnet-trace`. | +| `--trace-profile` | `cpu` | `cpu` (sampled thread time), `alloc` (allocation ticks), `contention` or `gc`. | +| `--out ` | | Append the result block to a file. | + +Environment variables prefixed `SERVER_` are passed to the server process without the prefix, +for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes. + +The result block reports client throughput and latency percentiles, the server deltas for the +window, and the server's allocated bytes and CPU time per request (or per notification). +Allocated bytes per operation are deterministic and the most reliable signal. On a shared +machine, throughput and latency vary from run to run, so compare builds with `ab.sh` rather +than with two separate runs. + +## Analyse a trace + +```bash +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll alloc trace.nettrace 40 +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll cpu trace.nettrace 40 Monitor.Enter_Slowpath +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll contention trace.nettrace 15 +``` + +The sampled thread-time profile also samples blocked threads. The `cpu` mode drops samples +whose leaf frame is a known wait (thread-pool semaphore, I/O completion poll, GC suspension), +so the remaining samples approximate on-CPU time. Spin-waiting in `Monitor.Enter_Slowpath` or +`Lock.TryEnterSlow` is real CPU and shows up there. + +## A/B comparison + +Build the baseline harness in a second worktree, then alternate the two builds: + +```bash +git worktree add --detach C:/uabase origin/master +cp -r perf C:/uabase/ && (cd C:/uabase && CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release) +cd perf && BASELINE=C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe ./ab.sh +pwsh -File ab-summary.ps1 +``` diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs new file mode 100644 index 0000000000..a1bc5d8c73 --- /dev/null +++ b/perf/ServerLoadHarness/Program.cs @@ -0,0 +1,647 @@ +// Server load / profiling harness. Not part of the product; see perf/README.md. +// Modes: +// server --port P [--diag 0|1] [--audit 0|1] [--threads N] +// Starts a ReferenceServer and reads commands from stdin: mark | report | quit +// run --scenario read|write|sub|browse [--sessions N] [--nodes K] [--inflight I] +// [--duration S] [--warmup S] [--security none|sign|encrypt] [--diag 0|1] [--audit 0|1] +// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] +// Spawns the server child, drives the load and prints a report of client latency +// and server-side CPU / GC / allocation deltas for the measurement window. + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using Microsoft.Extensions.Logging; +using Opc.Ua; +using Opc.Ua.Client; +using Opc.Ua.Client.TestFramework; +using Opc.Ua.Configuration; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; + +internal static class Program +{ + private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); + + private static string Arg(string name, string def) => s_args.TryGetValue(name, out string? v) ? v : def; + private static int ArgInt(string name, int def) => int.Parse(Arg(name, def.ToString(CultureInfo.InvariantCulture)), CultureInfo.InvariantCulture); + + public static async Task Main(string[] args) + { + for (int i = 1; i < args.Length; i++) + { + if (args[i].StartsWith("--", StringComparison.Ordinal)) + { + string key = args[i][2..]; + string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) ? args[++i] : "1"; + s_args[key] = value; + } + } + string mode = args.Length > 0 ? args[0] : "run"; + return mode switch + { + "server" => await RunServerAsync().ConfigureAwait(false), + _ => await RunDriverAsync(args).ConfigureAwait(false) + }; + } + + // ------------------------------------------------------------------ server + + private static ITelemetryContext CreateTelemetry(LogLevel level) => + DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); + + private static async Task RunServerAsync() + { + int port = ArgInt("port", 62555); + bool diag = ArgInt("diag", 0) != 0; + bool audit = ArgInt("audit", 0) != 0; + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + + string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); + var application = new ApplicationInstance(telemetry) + { + ApplicationName = "PerfReferenceServer", + ApplicationType = ApplicationType.Server + }; + string endpointUrl = $"opc.tcp://localhost:{port}/PerfReferenceServer"; + ArrayOf certs = ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( + "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", CertificateStoreType.Directory, pkiRoot); + ApplicationConfiguration config = await application + .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") + .SetMaxByteStringLength(4 * 1024 * 1024) + .SetMaxArrayLength(1024 * 1024) + .SetMaxMessageSize(16 * 1024 * 1024) + .AsServer([endpointUrl]) + .AddUnsecurePolicyNone() + .AddSignPolicies() + .AddSignAndEncryptPolicies() + .SetMaxChannelCount(2100) + .SetDiagnosticsEnabled(diag) + .SetAuditingEnabled(audit) + .SetShutdownDelay(0) + .AddSecurityConfiguration(certs, pkiRoot) + .SetAutoAcceptUntrustedCertificates(true) + .CreateAsync().ConfigureAwait(false); + config.ServerConfiguration!.MaxSessionCount = 2000; + config.ServerConfiguration.MaxSubscriptionCount = 20000; + config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; + config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); + config.ServerConfiguration.MinRequestThreadCount = 50; + + if (!await application.CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime).ConfigureAwait(false)) + { + Console.WriteLine("ERROR cert"); + return 1; + } + + var server = new ReferenceServer(telemetry); + server.TransportBindings = TestTransportBindings.WithAllSchemes(); + await application.StartAsync(server).ConfigureAwait(false); + Console.WriteLine($"READY {endpointUrl} {Environment.ProcessId}"); + + ServerStats mark = ServerStats.Capture(); + string? line; + while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) + { + switch (line.Trim()) + { + case "mark": + GC.Collect(); + GC.WaitForPendingFinalizers(); + mark = ServerStats.Capture(); + Console.WriteLine("MARKED"); + break; + case "report": + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + break; + case "quit": + await server.StopAsync().ConfigureAwait(false); + return 0; + } + } + return 0; + } + + private sealed record ServerStats(long Alloc, int G0, int G1, int G2, TimeSpan Pause, TimeSpan Cpu, long Contention, long Stamp, int Threads, long WorkingSet, long Heap) + { + public static ServerStats Capture() + { + using var p = Process.GetCurrentProcess(); + return new ServerStats( + GC.GetTotalAllocatedBytes(true), GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), + GC.GetTotalPauseDuration(), p.TotalProcessorTime, Monitor.LockContentionCount, Stopwatch.GetTimestamp(), + ThreadPool.ThreadCount, p.WorkingSet64, GC.GetGCMemoryInfo().HeapSizeBytes); + } + + public string Delta(ServerStats m) + { + double sec = Stopwatch.GetElapsedTime(m.Stamp, Stamp).TotalSeconds; + return string.Create(CultureInfo.InvariantCulture, + $"sec={sec:F2} allocMB={(Alloc - m.Alloc) / 1048576.0:F1} gen0={G0 - m.G0} gen1={G1 - m.G1} gen2={G2 - m.G2} pauseMs={(Pause - m.Pause).TotalMilliseconds:F0} cpuMs={(Cpu - m.Cpu).TotalMilliseconds:F0} cpuCores={(Cpu - m.Cpu).TotalSeconds / sec:F2} contention={Contention - m.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); + } + } + + // ------------------------------------------------------------------ driver + + private static async Task RunDriverAsync(string[] args) + { + string scenario = Arg("scenario", "read"); + int sessionsCount = ArgInt("sessions", 20); + int durationSec = ArgInt("duration", 20); + int warmupSec = ArgInt("warmup", 10); + int port = ArgInt("port", 62555 + Environment.ProcessId % 1000); + string security = Arg("security", "none"); + + string exe = Environment.ProcessPath!; + var psi = new ProcessStartInfo(exe, $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}") + { + RedirectStandardInput = true, + RedirectStandardOutput = true, + UseShellExecute = false + }; + foreach (System.Collections.DictionaryEntry kv in Environment.GetEnvironmentVariables()) + { + string k = (string)kv.Key; + if (k.StartsWith("SERVER_", StringComparison.Ordinal)) + { + psi.Environment[k[7..]] = (string?)kv.Value; + } + } + using Process server = Process.Start(psi)!; + var serverLines = new BlockingCollection(); + _ = Task.Run(() => + { + string? l; + while ((l = server.StandardOutput.ReadLine()) != null) + { + if (l.StartsWith("READY", StringComparison.Ordinal) || l.StartsWith("REPORT", StringComparison.Ordinal) || l.StartsWith("MARKED", StringComparison.Ordinal) || l.StartsWith("ERROR", StringComparison.Ordinal)) + { + serverLines.Add(l); + } + else if (s_args.ContainsKey("verbose")) + { + Console.WriteLine("[server] " + l); + } + } + serverLines.CompleteAdding(); + }); + string ready = serverLines.Take(); + if (!ready.StartsWith("READY", StringComparison.Ordinal)) + { + Console.WriteLine(ready); + return 1; + } + string url = ready.Split(' ')[1]; + Console.WriteLine($"server pid={server.Id} url={url}"); + + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; + await client.LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient").ConfigureAwait(false); + string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; + ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); + MessageSecurityMode wantedMode = security switch + { + "sign" => MessageSecurityMode.Sign, + "encrypt" => MessageSecurityMode.SignAndEncrypt, + _ => MessageSecurityMode.None + }; + var filtered = new List(); + foreach (EndpointDescription ep in endpoints) + { + if (ep.SecurityPolicyUri == policy && ep.SecurityMode == wantedMode) + { + filtered.Add(ep); + } + } + + var sessions = new ISession[sessionsCount]; + var connectSw = Stopwatch.StartNew(); + await Parallel.ForAsync(0, sessionsCount, new ParallelOptions { MaxDegreeOfParallelism = 32 }, async (i, ct) => + sessions[i] = await client.ConnectAsync(new Uri(url), policy, filtered).ConfigureAwait(false)).ConfigureAwait(false); + Console.WriteLine($"connected {sessionsCount} sessions ({security}) in {connectSw.ElapsedMilliseconds} ms"); + + Workload workload = scenario switch + { + "sub" => new SubscribeWorkload(sessions, ArgInt("nodes", 100), ArgInt("subs", 5), ArgInt("pub", 100), ArgInt("write-interval", 100)), + _ => new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)) + }; + await workload.SetupAsync().ConfigureAwait(false); + + using var stop = new CancellationTokenSource(); + Task run = workload.RunAsync(stop.Token); + await Task.Delay(TimeSpan.FromSeconds(warmupSec)).ConfigureAwait(false); + + Process? tracer = null; + string? trace = s_args.TryGetValue("trace", out string? t) ? t : null; + if (trace != null) + { + string profile = Arg("trace-profile", "cpu"); + string providers = profile switch + { + // allocation sampling (GCAllocationTick ~ every 100KB) + GC events + "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5 --clrevents gc --clreventlevel verbose", + "gc" => "--profile gc-verbose", + "contention" => "--providers Microsoft-Windows-DotNETRuntime:0x4000:4", + _ => "--profile dotnet-sampled-thread-time" + }; + tracer = Process.Start(new ProcessStartInfo("dotnet-trace", $"collect --process-id {server.Id} {providers} -o \"{trace}\" --duration 00:00:{durationSec:D2}") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }); + await Task.Delay(2000).ConfigureAwait(false); + } + + server.StandardInput.WriteLine("mark"); + Take(serverLines, "MARKED"); + workload.ResetStats(); + var sw = Stopwatch.StartNew(); + await Task.Delay(TimeSpan.FromSeconds(durationSec)).ConfigureAwait(false); + server.StandardInput.WriteLine("report"); + string report = Take(serverLines, "REPORT"); + double elapsed = sw.Elapsed.TotalSeconds; + string clientReport = workload.Report(elapsed); + stop.Cancel(); + try + { + await run.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + if (tracer != null) + { + await tracer.WaitForExitAsync().ConfigureAwait(false); + Console.WriteLine("trace: " + trace); + } + + var sb = new StringBuilder(); + sb.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionsCount} {string.Join(' ', args.Skip(1))}\n"); + sb.Append(" client: ").Append(clientReport).Append('\n'); + sb.Append(" server: ").Append(report[7..]).Append('\n'); + double ops = workload.Operations; + if (ops > 0) + { + ServerReport sr = ServerReport.Parse(report[7..]); + sb.Append(CultureInfo.InvariantCulture, $" per-op: allocB={sr.AllocMb * 1048576 / ops:F0} cpuUs={sr.CpuMs * 1000 / ops:F1} ({workload.OperationName})\n"); + } + Console.Write(sb.ToString()); + if (s_args.TryGetValue("out", out string? outFile)) + { + File.AppendAllText(outFile, sb.ToString()); + } + + await Parallel.ForEachAsync(sessions, async (s, ct) => + { + try + { + await s.CloseAsync(ct).ConfigureAwait(false); + } + catch + { + } + s.Dispose(); + }).ConfigureAwait(false); + server.StandardInput.WriteLine("quit"); + if (!server.WaitForExit(10000)) + { + server.Kill(); + } + return 0; + } + + private static string Take(BlockingCollection lines, string prefix) + { + while (true) + { + string l = lines.Take(); + if (l.StartsWith(prefix, StringComparison.Ordinal)) + { + return l; + } + } + } + + private sealed record ServerReport(double AllocMb, double CpuMs) + { + public static ServerReport Parse(string s) + { + var d = s.Split(' ').Select(p => p.Split('=')).Where(p => p.Length == 2).ToDictionary(p => p[0], p => p[1]); + return new ServerReport(double.Parse(d["allocMB"], CultureInfo.InvariantCulture), double.Parse(d["cpuMs"], CultureInfo.InvariantCulture)); + } + } + + internal static List GetNodes(ISession session, int count, Type? onlyType = null) + { + var result = new List(); + foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + { + if (onlyType != null && type != onlyType) + { + continue; + } + foreach (ExpandedNodeId e in ids) + { + if (result.Count == count) + { + return result; + } + result.Add(ExpandedNodeId.ToNodeId(e, session.NamespaceUris)); + } + } + return result; + } +} + +internal abstract class Workload +{ + public abstract Task SetupAsync(); + public abstract Task RunAsync(CancellationToken ct); + public abstract void ResetStats(); + public abstract string Report(double seconds); + public abstract double Operations { get; } + public abstract string OperationName { get; } + + protected static string Percentiles(List ticksList) + { + if (ticksList.Count == 0) + { + return "n=0"; + } + long[] a = [.. ticksList]; + Array.Sort(a); + double Ms(double q) => a[Math.Min(a.Length - 1, (int)(q * a.Length))] * 1000.0 / Stopwatch.Frequency; + return string.Create(CultureInfo.InvariantCulture, + $"n={a.Length} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); + } +} + +internal sealed class RequestWorkload : Workload +{ + private readonly string m_kind; + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_inflight; + private readonly ConcurrentBag> m_lat = new(); + private readonly ThreadLocal> m_local; + private volatile int m_epoch; + private long m_requests; + private long m_errors; + private ArrayOf m_read; + private ArrayOf m_write; + private ArrayOf m_browse; + + public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) + { + m_kind = kind; + m_sessions = sessions; + m_nodes = nodes; + m_inflight = inflight; + m_local = new ThreadLocal>(() => + { + var l = new List(1 << 16); + m_lat.Add(l); + return l; + }, false); + } + + public override double Operations => Interlocked.Read(ref m_requests); + public override string OperationName => "request"; + + public override Task SetupAsync() + { + List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); + m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); + m_write = nodes.Select(n => new WriteValue { NodeId = n, AttributeId = Attributes.Value, Value = new DataValue(new Variant(42)) }).ToArray(); + m_browse = new[] { new BrowseDescription { NodeId = Opc.Ua.ObjectIds.Server, BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All } }; + Console.WriteLine($"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session"); + return Task.CompletedTask; + } + + public override async Task RunAsync(CancellationToken ct) + { + var tasks = new List(); + foreach (ISession s in m_sessions) + { + for (int i = 0; i < m_inflight; i++) + { + tasks.Add(Task.Run(() => LoopAsync(s, ct), CancellationToken.None)); + } + } + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + private async Task LoopAsync(ISession s, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + long start = Stopwatch.GetTimestamp(); + int epoch = m_epoch; + try + { + switch (m_kind) + { + case "write": + await s.WriteAsync(null, m_write, ct).ConfigureAwait(false); + break; + case "browse": + await s.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); + break; + default: + await s.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); + break; + } + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + if (Interlocked.Increment(ref m_errors) < 5) + { + Console.WriteLine("error: " + ex.Message); + } + continue; + } + long el = Stopwatch.GetTimestamp() - start; + if (epoch == m_epoch) + { + List l = m_local.Value!; + lock (l) + { + l.Add(el); + } + Interlocked.Increment(ref m_requests); + } + } + } + + public override void ResetStats() + { + m_epoch++; + foreach (List l in m_lat) + { + lock (l) + { + l.Clear(); + } + } + Interlocked.Exchange(ref m_requests, 0); + Interlocked.Exchange(ref m_errors, 0); + } + + public override string Report(double seconds) + { + var all = new List(); + foreach (List l in m_lat) + { + lock (l) + { + all.AddRange(l); + } + } + long req = Interlocked.Read(ref m_requests); + return string.Create(CultureInfo.InvariantCulture, + $"req/s={req / seconds:F0} values/s={req * (m_kind == "browse" ? 1 : m_nodes) / seconds:F0} errors={m_errors} latency {Percentiles(all)}"); + } +} + +internal sealed class SubscribeWorkload : Workload +{ + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_subs; + private readonly int m_pub; + private readonly int m_writeInterval; + private readonly long[] m_writeTicks = new long[1 << 16]; + private readonly ConcurrentBag> m_lat = new(); + private readonly ThreadLocal> m_local; + private long m_notifications; + private long m_writes; + private int m_counter; + private ISession? m_writer; + private ArrayOf m_nodeIds; + + public SubscribeWorkload(ISession[] sessions, int nodes, int subs, int pub, int writeInterval) + { + m_sessions = sessions; + m_nodes = nodes; + m_subs = subs; + m_pub = pub; + m_writeInterval = writeInterval; + m_local = new ThreadLocal>(() => + { + var l = new List(1 << 16); + m_lat.Add(l); + return l; + }, false); + } + + public override double Operations => Interlocked.Read(ref m_notifications); + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); + m_writer = m_sessions[0]; + var sw = Stopwatch.StartNew(); + await Parallel.ForEachAsync(m_sessions, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (session, ct) => + { + for (int j = 0; j < m_subs; j++) + { + var subscription = new Subscription(session.DefaultSubscription) + { + PublishingInterval = m_pub, + KeepAliveCount = 10, + LifetimeCount = 100 + }; + foreach (NodeId n in m_nodeIds) + { + subscription.AddItem(new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = n, + AttributeId = Attributes.Value, + MonitoringMode = MonitoringMode.Reporting, + SamplingInterval = 0, + QueueSize = 1 + }); + } + subscription.FastDataChangeCallback = (sub, notification, _) => + { + long now = Stopwatch.GetTimestamp(); + List l = m_local.Value!; + int count = 0; + lock (l) + { + foreach (MonitoredItemNotification item in notification.MonitoredItems) + { + if (item.Value.WrappedValue.TryGetValue(out int v) && v > 0) + { + long w = Volatile.Read(ref m_writeTicks[v & 0xFFFF]); + if (w != 0) + { + l.Add(now - w); + } + } + count++; + } + } + Interlocked.Add(ref m_notifications, count); + }; + session.AddSubscription(subscription); + await subscription.CreateAsync(ct).ConfigureAwait(false); + } + }).ConfigureAwait(false); + Console.WriteLine($"sub: {m_sessions.Length} sessions x {m_subs} subs x {m_nodeIds.Count} items = {m_sessions.Length * m_subs * m_nodeIds.Count} items, pub {m_pub} ms, write every {m_writeInterval} ms; setup {sw.ElapsedMilliseconds} ms"); + } + + public override async Task RunAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); + while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) + { + int v = Interlocked.Increment(ref m_counter); + var values = new WriteValue[m_nodeIds.Count]; + for (int i = 0; i < values.Length; i++) + { + values[i] = new WriteValue { NodeId = m_nodeIds[i], AttributeId = Attributes.Value, Value = new DataValue(new Variant(v)) }; + } + Volatile.Write(ref m_writeTicks[v & 0xFFFF], Stopwatch.GetTimestamp()); + await m_writer!.WriteAsync(null, values, ct).ConfigureAwait(false); + Interlocked.Increment(ref m_writes); + } + } + + public override void ResetStats() + { + foreach (List l in m_lat) + { + lock (l) + { + l.Clear(); + } + } + Interlocked.Exchange(ref m_notifications, 0); + Interlocked.Exchange(ref m_writes, 0); + } + + public override string Report(double seconds) + { + var all = new List(); + foreach (List l in m_lat) + { + lock (l) + { + all.AddRange(l); + } + } + long n = Interlocked.Read(ref m_notifications); + long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subs * m_nodeIds.Count; + return string.Create(CultureInfo.InvariantCulture, + $"notif/s={n / seconds:F0} received/expected={(expected == 0 ? 0 : (double)n / expected):P1} write->notify latency {Percentiles(all)}"); + } +} diff --git a/perf/ServerLoadHarness/ServerLoadHarness.csproj b/perf/ServerLoadHarness/ServerLoadHarness.csproj new file mode 100644 index 0000000000..1d238e7b18 --- /dev/null +++ b/perf/ServerLoadHarness/ServerLoadHarness.csproj @@ -0,0 +1,20 @@ + + + Exe + net10.0 + enable + enable + false + false + true + true + $(NoWarn);CS1591;CA1303;CA2007;CA1031;CA2000;CA1849;CA5394;RCS1090;CA1515;CA1305;CA1307;CA1310;IDE0005;CA1062;CA1822;CA1859;CA1861;CS0618;RCS1163;CA1002;CA1812;CA1852;CA1050;CA2201;CA1001 + false + false + none + + + + + + diff --git a/perf/TraceAnalyzer/Program.cs b/perf/TraceAnalyzer/Program.cs new file mode 100644 index 0000000000..a8002ec6eb --- /dev/null +++ b/perf/TraceAnalyzer/Program.cs @@ -0,0 +1,174 @@ +// Trace analyzer for perf/ServerLoadHarness traces. Not part of the product; see perf/README.md. +// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped), exclusive/inclusive, callers of focus +// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, by first Opc.Ua frame, and stacks +// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack +using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tracing.Etlx; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +string mode = args[0]; +string file = args[1]; +int top = args.Length > 2 ? int.Parse(args[2]) : 40; +string focus = args.Length > 3 ? args[3] : null; +string[] s_blocked = { "LowLevelLifoSemaphore.Wait", "WaitForSignal", "WorkerThreadStart", "IOCompletionPoller.Poll", "WaitOneNoCheck", "Thread.Sleep", "PollGCWorker|70_0", "WaitMultipleIgnoringSyncContext", "Monitor.Wait", "!?" }; + +string etlx = TraceLog.CreateFromEventPipeDataFile(file, null, new TraceLogOptions { ContinueOnError = true }); +using var log = new TraceLog(etlx); + +static string Name(TraceCallStack s) +{ + var m = s.CodeAddress.FullMethodName; + if (string.IsNullOrEmpty(m)) + { + m = s.CodeAddress.ModuleName + "!?"; + } + return m; +} + +static string Short(string m) +{ + int p = m.IndexOf('('); + return p > 0 ? m[..p] : m; +} + +static bool IsOurs(string m) => m.StartsWith("Opc.Ua", StringComparison.Ordinal) || m.StartsWith("Quickstarts", StringComparison.Ordinal); + +if (mode == "contention") +{ + var byStack = new Dictionary(); + int total = 0; + foreach (TraceEvent e in log.Events) + { + if (e.EventName == null || !e.EventName.StartsWith("Contention/Start")) continue; + total++; + var chain = new List(); + for (TraceCallStack f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) { string n = Short(Name(f)); if (IsOurs(n) || chain.Count == 0) chain.Add(n); } + string k = string.Join("\n ", chain); + byStack[k] = byStack.GetValueOrDefault(k) + 1; + } + Console.WriteLine($"contention events: {total}"); + foreach (var kv in byStack.OrderByDescending(k => k.Value).Take(top)) Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + return; +} +if (mode == "cpu") +{ + var excl = new Dictionary(); + var incl = new Dictionary(); + var firstOurs = new Dictionary(); + var callers = new Dictionary(); + int total = 0, all = 0; + foreach (TraceEvent e in log.Events) + { + if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") + { + continue; + } + all++; + object type = e.PayloadByName("Type"); + if (type != null && Convert.ToInt32(type) != 2) + { + continue; // not running managed code + } + TraceCallStack s = e.CallStack(); + if (s == null) + { + continue; + } + string leaf = Short(Name(s)); + if (s_blocked.Any(b => leaf.EndsWith(b))) continue; + total++; + excl[leaf] = excl.GetValueOrDefault(leaf) + 1; + var seen = new HashSet(); + bool gotOurs = false; + bool focusHit = false; + for (TraceCallStack f = s; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (seen.Add(n)) + { + incl[n] = incl.GetValueOrDefault(n) + 1; + } + if (!gotOurs && IsOurs(n)) + { + gotOurs = true; + firstOurs[n] = firstOurs.GetValueOrDefault(n) + 1; + } + if (focus != null && !focusHit && n.Contains(focus)) + { + focusHit = true; + string c = f.Caller != null ? Short(Name(f.Caller)) : ""; + callers[c] = callers.GetValueOrDefault(c) + 1; + } + } + } + Console.WriteLine($"samples: managed-running={total} all={all}"); + void Dump(string title, Dictionary d, int n) + { + Console.WriteLine($"--- {title}"); + foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + { + Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + } + } + Dump("exclusive", excl, top); + Dump("first Opc.Ua frame (exclusive-ish)", firstOurs, top); + Dump("inclusive (Opc.Ua only)", incl.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value), top); + if (focus != null) + { + Dump("callers of " + focus, callers, top); + } +} +else +{ + var byType = new Dictionary(); + var byFrame = new Dictionary(); + var byStack = new Dictionary(); + double total = 0; + foreach (TraceEvent e in log.Events) + { + if (e is not GCAllocationTickTraceData a) + { + continue; + } + double amount = a.AllocationAmount64; + total += amount; + string t = a.TypeName ?? "?"; + byType[t] = byType.GetValueOrDefault(t) + amount; + TraceCallStack s = e.CallStack(); + string frame = "?"; + var chain = new List(); + for (TraceCallStack f = s; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (IsOurs(n)) + { + if (frame == "?") + { + frame = n; + } + if (chain.Count < 6) + { + chain.Add(n); + } + } + } + byFrame[t + " <- " + frame] = byFrame.GetValueOrDefault(t + " <- " + frame) + amount; + if (focus == null || t.Contains(focus)) + { + string k = t + "\n " + string.Join("\n ", chain); + byStack[k] = byStack.GetValueOrDefault(k) + amount; + } + } + Console.WriteLine($"sampled alloc total: {total / 1048576:F1} MB"); + void Dump(string title, Dictionary d, int n) + { + Console.WriteLine($"--- {title}"); + foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + { + Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + } + } + Dump("by type", byType, top); + Dump("by type <- first Opc.Ua frame", byFrame, top); + Dump("stacks", byStack, top / 2); +} diff --git a/perf/TraceAnalyzer/TraceAnalyzer.csproj b/perf/TraceAnalyzer/TraceAnalyzer.csproj new file mode 100644 index 0000000000..e2529b9c27 --- /dev/null +++ b/perf/TraceAnalyzer/TraceAnalyzer.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + disable + enable + false + false + false + none + $(NoWarn);CS1591;CA1305;CA1307;CA1310;CA1849;RCS1007;IDE0011 + + + + + diff --git a/perf/ab-summary.ps1 b/perf/ab-summary.ps1 new file mode 100644 index 0000000000..6f2ca38803 --- /dev/null +++ b/perf/ab-summary.ps1 @@ -0,0 +1,25 @@ +param($file = "ab-results.txt") +$rows = @() +$cur = $null +foreach ($l in Get-Content $file) { + if ($l -match '^### side=(\w) round=(\d) (.*)$') { $cur = @{ side = $Matches[1]; scen = $Matches[3].Trim() }; continue } + if ($null -eq $cur) { continue } + if ($l -match 'req/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'notif/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'p50=([\d.]+)ms p90=([\d.]+)ms p99=([\d.]+)ms') { $cur.p50 = [double]$Matches[1]; $cur.p90 = [double]$Matches[2]; $cur.p99 = [double]$Matches[3] } + if ($l -match 'pauseMs=(\d+)') { $cur.pause = [double]$Matches[1] } + if ($l -match 'contention=(\d+)') { $cur.cont = [double]$Matches[1] } + if ($l -match 'per-op: allocB=(\d+) cpuUs=([\d.]+)') { $cur.alloc = [double]$Matches[1]; $cur.cpu = [double]$Matches[2]; $rows += [pscustomobject]$cur; $cur = $null } +} +function Med($xs) { $s = @($xs | Sort-Object); if ($s.Count -eq 0) { return [double]::NaN }; return $s[[int][Math]::Floor($s.Count / 2)] } +$out = @() +foreach ($g in ($rows | Group-Object scen)) { + $a = $g.Group | Where-Object side -eq 'A'; $b = $g.Group | Where-Object side -eq 'B' + $o = [ordered]@{ scenario = $g.Name; n = "$(@($a).Count)/$(@($b).Count)" } + foreach ($m in 'tput', 'p50', 'p90', 'p99', 'alloc', 'cpu', 'pause', 'cont') { + $ma = Med ($a | ForEach-Object { $_.$m }); $mb = Med ($b | ForEach-Object { $_.$m }) + $o["$m"] = "{0:G4} -> {1:G4} ({2:+0;-0}%)" -f $ma, $mb, (100 * ($mb - $ma) / $ma) + } + $out += [pscustomobject]$o +} +$out | Format-List diff --git a/perf/ab.sh b/perf/ab.sh new file mode 100644 index 0000000000..3f26ae6dcd --- /dev/null +++ b/perf/ab.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# BASELINE: harness built in a second worktree of the commit to compare against. +A=${BASELINE:-C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe} +B=./ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe +O=ab-results.txt; rm -f $O +while IFS= read -r s; do + for r in 1 2 3; do + for side in A B; do + exe=$A; [ $side = B ] && exe=$B + echo "### side=$side round=$r $s" >> $O + timeout 300 $exe run $s --sessions 20 --duration 12 --warmup 6 --out $O >/dev/null 2>&1 + done + done +done <<'S' +--scenario read --nodes 1 +--scenario read --nodes 100 +--scenario write --nodes 100 +--scenario browse +--scenario read --nodes 100 --security encrypt +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 +S +echo done >> $O From 9068684c39aefee61af99b781f99ab527dc1a97e Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 07:19:00 +0200 Subject: [PATCH 03/16] Server: validate node permissions without allocating NodeMetadata Read, Write, Browse and Call validate every node they touch, and the dispatcher discarded the NodeMetadata (248 B, plus copies of the role permission arrays) right after the check. Browse also read 13 attributes per reference target only to check its RolePermissions. The role-permission and access-restriction validators now run on an internal PermissionMetadata struct holding exactly the attributes they read; the NodeMetadata overloads build it and delegate, so the checks are unchanged. AsyncCustomNodeManager fills the struct on the stack from the same three attributes and namespace defaults that GetPermissionMetadataAsync reads. The dispatcher uses it when the metadata is not needed, and Browse's reference filter uses it instead of GetNodeMetadataAsync. Unresolved handles still take the NodeMetadata path with its fallbacks. Read 100 values: alloc/request 120.6 -> 95.7 KB, ~21k -> 26k req/s; Browse: 67.6 -> 51.8 KB, 26.7k -> 29.8k req/s. Co-Authored-By: Claude Opus 5.5 --- .../NodeManager/AsyncCustomNodeManager.cs | 154 +++++++++++++++++- .../MasterNodeManager.ServiceDispatch.cs | 40 ++++- .../NodeManagerServiceDispatcher.cs | 27 +++ .../NodeManager/PermissionMetadata.cs | 104 ++++++++++++ 4 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs diff --git a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs index 88cb9791b8..34c27617f1 100644 --- a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs @@ -9204,6 +9204,25 @@ public virtual async ValueTask ValidateRolePermissionsAsync( return StatusCodes.Good; } + // Browse validates every reference target: read only the permission + // attributes instead of the full node metadata. + if (nodeManager is AsyncCustomNodeManager customNodeManager) + { + (bool handled, ServiceResult? verdict) = await customNodeManager.TryValidatePermissionsAsync( + operationContext, + nodeHandle, + requestedPermission, + null, + permissionsOnly: true, + logger: null, + validateAccessRestrictions: false, + cancellationToken).ConfigureAwait(false); + if (handled) + { + return verdict!; + } + } + NodeMetadata nodeMetadata = await nodeManager.GetNodeMetadataAsync( operationContext, nodeHandle, @@ -10460,6 +10479,133 @@ private async ValueTask SetDefaultPermissionsAsync( NodeMetadata metadata, CancellationToken cancellationToken = default) { + (AccessRestrictionType defaultAccessRestrictions, + ArrayOf defaultRolePermissions, + ArrayOf defaultUserRolePermissions) = + await GetDefaultPermissionsAsync(systemContext, target, cancellationToken).ConfigureAwait(false); + metadata.DefaultAccessRestrictions = defaultAccessRestrictions; + metadata.DefaultRolePermissions = defaultRolePermissions; + metadata.DefaultUserRolePermissions = defaultUserRolePermissions; + } + + /// + /// Validates the role permissions and access restrictions of a node for the requested + /// permission, reading the same attributes as + /// without materializing a . + /// + /// + /// Handled is false when the handle does not resolve to a node of this + /// node manager; the caller then takes the path, which has + /// the fallbacks for that case. + /// + internal async ValueTask<(bool Handled, ServiceResult? Result)> TryValidatePermissionsAsync( + OperationContext context, + object targetHandle, + PermissionType requestedPermission, + Dictionary? uniqueNodesServiceAttributesCache, + bool permissionsOnly, + ILogger? logger, + bool validateAccessRestrictions, + CancellationToken cancellationToken = default) + { + ServerSystemContext systemContext = GetOperationSystemContext(context); + + NodeHandle? handle = IsHandleInNamespace(targetHandle); + if (handle == null) + { + return (false, null); + } + + NodeState? target = await ValidateNodeAsync(systemContext, handle, null!, cancellationToken).ConfigureAwait(false); + if (target == null) + { + return (false, null); + } + + Variant[]? values = null; + if (uniqueNodesServiceAttributesCache != null) + { + NodeId key = handle.NodeId; + if (!uniqueNodesServiceAttributesCache.TryGetValue(key, out values) || + values.Length != 3) + { + values = new Variant[3]; + ReadAndCacheValidationAttributes( + uniqueNodesServiceAttributesCache, + systemContext, + target, + key, + ref values); + } + } + else if (permissionsOnly) + { + values = new Variant[3]; + ReadValidationAttributes(systemContext, target, ref values); + } + + // the same conversions as SetAccessAndRolePermissions; the arrays are only read + // for the verdict, so they are not copied. + AccessRestrictionType accessRestrictions = AccessRestrictionType.None; + ArrayOf rolePermissions = default; + ArrayOf userRolePermissions = default; + if (values != null) + { + if (values[0].TryGetValue(out ushort restrictions)) + { + accessRestrictions = (AccessRestrictionType)restrictions; + } + if (values[1].TryGetStructure(out ArrayOf roles)) + { + rolePermissions = roles; + } + if (values[2].TryGetStructure(out ArrayOf userRoles)) + { + userRolePermissions = userRoles; + } + } + + (AccessRestrictionType defaultAccessRestrictions, + ArrayOf defaultRolePermissions, + ArrayOf defaultUserRolePermissions) = + await GetDefaultPermissionsAsync(systemContext, target, cancellationToken).ConfigureAwait(false); + + var permissions = new PermissionMetadata( + target.NodeId, + target.IsPartOfTypeHierarchy, + accessRestrictions, + defaultAccessRestrictions, + rolePermissions, + defaultRolePermissions, + userRolePermissions, + defaultUserRolePermissions); + + ServiceResult result = MasterNodeManager.ValidateRolePermissions( + context, + permissions, + requestedPermission, + logger); + return (true, validateAccessRestrictions && ServiceResult.IsGood(result) + ? MasterNodeManager.ValidateAccessRestrictions(context, permissions, requestedPermission) + : result); + } + + /// + /// Reads the namespace default values for DefaultAccessRestrictions, DefaultRolePermissions + /// and DefaultUserRolePermissions of the node's namespace. + /// + private async ValueTask<( + AccessRestrictionType DefaultAccessRestrictions, + ArrayOf DefaultRolePermissions, + ArrayOf DefaultUserRolePermissions)> GetDefaultPermissionsAsync( + ServerSystemContext systemContext, + NodeState target, + CancellationToken cancellationToken = default) + { + AccessRestrictionType defaultAccessRestrictions = AccessRestrictionType.None; + ArrayOf defaultRolePermissions = default; + ArrayOf defaultUserRolePermissions = default; + // check if NamespaceMetadata is defined for NamespaceIndex of the node. NamespaceMetadataState? namespaceMetadataState = await Server.NodeManager.ConfigurationNodeManager!.GetNamespaceMetadataStateAsync(target.NodeId.NamespaceIndex, cancellationToken) @@ -10482,7 +10628,7 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull) { - metadata.DefaultAccessRestrictions = + defaultAccessRestrictions = value.GetEnumeration(); } } @@ -10499,7 +10645,7 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull && value.TryGetStructure(out ArrayOf rolePermissions)) { - metadata.DefaultRolePermissions = rolePermissions; + defaultRolePermissions = rolePermissions; } } @@ -10515,10 +10661,12 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull && value.TryGetStructure(out ArrayOf userRolePermissions)) { - metadata.DefaultUserRolePermissions = userRolePermissions; + defaultUserRolePermissions = userRolePermissions; } } } + + return (defaultAccessRestrictions, defaultRolePermissions, defaultUserRolePermissions); } /// diff --git a/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs b/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs index 7f5196bce6..5ec88af821 100644 --- a/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs +++ b/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs @@ -874,7 +874,7 @@ protected ValueTask ValidatePermissionsAsync( /// hierarchy visibility exemption. /// private static bool IsTypeHierarchyVisibilityRequest( - NodeMetadata nodeMetadata, + in PermissionMetadata nodeMetadata, PermissionType requestedPermission) { return nodeMetadata.IsPartOfTypeHierarchy && @@ -910,6 +910,20 @@ protected internal static ServiceResult ValidateAccessRestrictions( OperationContext context, NodeMetadata nodeMetadata, PermissionType requestedPermission) + { + return ValidateAccessRestrictions( + context, + PermissionMetadata.From(nodeMetadata), + requestedPermission); + } + + /// + /// Validate the AccessRestrictions attribute for the requested permission. + /// + internal static ServiceResult ValidateAccessRestrictions( + OperationContext context, + in PermissionMetadata nodeMetadata, + PermissionType requestedPermission) { ServiceResult serviceResult = StatusCodes.Good; @@ -993,7 +1007,29 @@ protected internal static ServiceResult ValidateRolePermissions( PermissionType requestedPermission, ILogger? logger = null) { - if (nodeMetadata == null || requestedPermission == PermissionType.None) + if (nodeMetadata == null) + { + // no permission is required hence the validation passes + return StatusCodes.Good; + } + + return ValidateRolePermissions( + context, + PermissionMetadata.From(nodeMetadata), + requestedPermission, + logger); + } + + /// + /// Validates the role permissions + /// + internal static ServiceResult ValidateRolePermissions( + OperationContext context, + in PermissionMetadata nodeMetadata, + PermissionType requestedPermission, + ILogger? logger = null) + { + if (requestedPermission == PermissionType.None) { // no permission is required hence the validation passes return StatusCodes.Good; diff --git a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs index 251eeda8d1..a98a50f707 100644 --- a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs +++ b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs @@ -3851,6 +3851,33 @@ internal async ValueTask ValidatePermissionsAsync( return (StatusCodes.Good, null); } + // Most callers only need the verdict: validate on the stack instead of + // materializing NodeMetadata for every node of the request. + if (!metadataRequired && nodeManager is AsyncCustomNodeManager customNodeManager) + { + try + { + (bool handled, ServiceResult? verdict) = await customNodeManager.TryValidatePermissionsAsync( + context, + nodeHandle, + requestedPermission, + uniqueNodesServiceAttributes, + permissionsOnly, + m_logger, + validateAccessRestrictions: true, + cancellationToken) + .ConfigureAwait(false); + if (handled) + { + return (verdict!, null); + } + } + catch (ServiceResultException exception) + { + return (new ServiceResult(exception), null); + } + } + // First attempt to retrieve just the Permission metadata with or without cache optimization // If it happens that nodemanager does not fully implement GetPermissionMetadata, // fallback to GetNodeMetadataAsync diff --git a/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs b/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs new file mode 100644 index 0000000000..3d8cff7236 --- /dev/null +++ b/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs @@ -0,0 +1,104 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +namespace Opc.Ua.Server +{ + /// + /// The attributes the role-permission and access-restriction checks read: + /// the subset of a permission check needs. + /// + /// + /// Read, Write, Browse and Call validate every node they touch. Filling this + /// value on the stack spares the per-node object when + /// the caller only needs the verdict. + /// + internal readonly struct PermissionMetadata + { + public PermissionMetadata( + NodeId nodeId, + bool isPartOfTypeHierarchy, + AccessRestrictionType accessRestrictions, + AccessRestrictionType defaultAccessRestrictions, + ArrayOf rolePermissions, + ArrayOf defaultRolePermissions, + ArrayOf userRolePermissions, + ArrayOf defaultUserRolePermissions) + { + NodeId = nodeId; + IsPartOfTypeHierarchy = isPartOfTypeHierarchy; + AccessRestrictions = accessRestrictions; + DefaultAccessRestrictions = defaultAccessRestrictions; + RolePermissions = rolePermissions; + DefaultRolePermissions = defaultRolePermissions; + UserRolePermissions = userRolePermissions; + DefaultUserRolePermissions = defaultUserRolePermissions; + } + + /// + /// Takes the permission attributes of node metadata. + /// + public static PermissionMetadata From(NodeMetadata metadata) + { + return new PermissionMetadata( + metadata.NodeId, + metadata.IsPartOfTypeHierarchy, + metadata.AccessRestrictions, + metadata.DefaultAccessRestrictions, + metadata.RolePermissions, + metadata.DefaultRolePermissions, + metadata.UserRolePermissions, + metadata.DefaultUserRolePermissions); + } + + /// + public NodeId NodeId { get; } + + /// + public bool IsPartOfTypeHierarchy { get; } + + /// + public AccessRestrictionType AccessRestrictions { get; } + + /// + public AccessRestrictionType DefaultAccessRestrictions { get; } + + /// + public ArrayOf RolePermissions { get; } + + /// + public ArrayOf DefaultRolePermissions { get; } + + /// + public ArrayOf UserRolePermissions { get; } + + /// + public ArrayOf DefaultUserRolePermissions { get; } + } +} From 936b28d76059ae669a87b52b87b02a64ed093b31 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:08 +0200 Subject: [PATCH 04/16] Server: expose channel certificates as read-only ByteString; halve session lookup locking SecureChannelContext.ClientChannelCertificate, ServerChannelCertificate and ChannelThumbprint are ByteString instead of byte[]. The TCP listener now shares one cached DER copy per channel with every request, and a mutable array handed to request handlers or IResourceIsolationClassifier code could have altered the evidence later requests are checked against. A ByteString is a read-only view, so the buffer is shared safely without copying. The constructor keeps its byte[] parameters and wraps them; a missing value is IsNull, so the session signature data (which skips a null certificate but hashes an empty one) is unchanged. Readers in Server, GDS, LDS and the Redundancy session mirror are migrated; the mirrored activation state keeps recording an empty, not null, certificate. MigrationGuide.md lists the mapping for application code. SessionManager.TryGetSessionContext runs at admission and at dispatch of every request and took the global bindings lock twice per call. The candidate snapshot now reads the concurrent session index and the activation-state table without the lock; the validating check under the lock still confirms the binding is the committed one. Co-Authored-By: Claude Opus 5.5 --- docs/MigrationGuide.md | 26 ++++++ .../Stack/Server/SecureChannelContext.cs | 31 +++++-- .../Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs | 3 +- .../AuthorizationHelper.cs | 9 +- src/Opc.Ua.Lds.Server/LdsServer.cs | 6 +- .../Sessions/DistributedSessionManager.cs | 15 +++- .../DefaultServerResourceIsolationProvider.cs | 12 +-- .../Session/ChannelEvidenceExtensions.cs | 48 ++++++++++ src/Opc.Ua.Server/Session/Session.cs | 30 +++---- src/Opc.Ua.Server/Session/SessionManager.cs | 44 ++++----- .../Session/SessionSecurityPolicyHelper.cs | 6 +- .../Stack/Server/SecureChannelContextTests.cs | 89 +++++++++++++++++++ ...istributedSessionMirrorIntegrationTests.cs | 2 +- .../ServerFixtureUtils.cs | 6 +- .../RuntimeResourceIsolationTests.cs | 2 +- .../EphemeralNonceLifetimeRegressionTests.cs | 4 +- .../SessionRestoreTests.cs | 2 +- .../SessionSecurityTests.cs | 24 ++--- 18 files changed, 275 insertions(+), 84 deletions(-) create mode 100644 src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs create mode 100644 tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index b9b2302357..9ce6b5f1fa 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -33,6 +33,7 @@ covers cross-cutting changes. - [Transport resource limits](#transport-resource-limits) - [Migrating channel subclasses that override HandleIncomingMessage](#migrating-channel-subclasses-that-override-handleincomingmessage) - [Migrating custom IUserDatabase implementations](#migrating-custom-iuserdatabase-implementations) +- [Reading certificates from SecureChannelContext](#reading-certificates-from-securechannelcontext) - [Write service Value semantics follow OPC 10000-3 and OPC 10000-4](#write-service-value-semantics-follow-opc-10000-3-and-opc-10000-4) - [Address space permissions, Call, NodeManagement and locales follow OPC 10000-3](#address-space-permissions-call-nodemanagement-and-locales-follow-opc-10000-3) - [Migrating from 1.05.377 to 1.05.378](#migrating-from-105377-to-105378) @@ -908,6 +909,31 @@ and no longer keeps metadata only in memory, so a store that cannot persist metadata should reject the write by returning `false` rather than silently accepting it. +## Reading certificates from SecureChannelContext + +`SecureChannelContext.ClientChannelCertificate`, `ServerChannelCertificate` and +`ChannelThumbprint` are `ByteString` instead of `byte[]`. The transport now shares +one buffer of a channel's certificates with every request on that channel instead +of copying the DER encoding per request, and a `ByteString` is a read-only view of +it, so a request handler or an `IResourceIsolationClassifier` cannot alter the +evidence later requests are checked against. + +The constructor still takes `byte[]` and wraps it without copying. A missing +value is `IsNull`; an empty array stays empty, not null. + +| Before | After | +|---|---| +| `context.ClientChannelCertificate == null` | `context.ClientChannelCertificate.IsNull` | +| `context.ClientChannelCertificate?.Length > 0` | `context.ClientChannelCertificate.Length > 0` | +| `context.ClientChannelCertificate.AsSpan()` | `context.ClientChannelCertificate.Span` | +| `Certificate.FromRawData(context.ClientChannelCertificate)` | `Certificate.FromRawData(context.ClientChannelCertificate.Memory)` | +| `context.ClientChannelCertificate.ToByteString()` | `context.ClientChannelCertificate` | +| passing the array to an API that takes `byte[]?` | `value.IsNull ? null : value.ToArray()` | + +Keep a missing certificate null when converting: the session signature helpers +hash an empty array but skip a null one, so substituting `ToArray()` for a null +value changes the signature data. + ## ContentFilter NULL semantics follow OPC 10000-4 1.05.07 `FilterEvaluator` applies the NULL rules of diff --git a/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs b/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs index ec02e0ae56..62e62ba1c9 100644 --- a/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs +++ b/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs @@ -58,9 +58,11 @@ public SecureChannelContext( SecureChannelId = secureChannelId; EndpointDescription = endpointDescription; MessageEncoding = messageEncoding; - ClientChannelCertificate = clientChannelCertificate; - ServerChannelCertificate = serverChannelCertificate; - ChannelThumbprint = channelThumbprint; + // Wrapped, not copied: the transport passes buffers it owns for the life of + // the channel, and ByteString hands out read-only views only. + ClientChannelCertificate = clientChannelCertificate == null ? default : new ByteString(clientChannelCertificate); + ServerChannelCertificate = serverChannelCertificate == null ? default : new ByteString(serverChannelCertificate); + ChannelThumbprint = channelThumbprint == null ? default : new ByteString(channelThumbprint); PeerAddress = peerAddress; } @@ -88,17 +90,30 @@ public SecureChannelContext( /// /// The unique hash for the secure channel calculated during channel creation. /// - public byte[]? ChannelThumbprint { get; } + /// + /// Read-only: the transport shares one buffer with every request of the channel. + /// when the channel has no thumbprint. + /// + public ByteString ChannelThumbprint { get; } /// - /// The client certificate used to establsih the secure channel. + /// The DER encoded client certificate used to establish the secure channel. /// - public byte[]? ClientChannelCertificate { get; } + /// + /// Read-only: the transport shares one buffer with every request of the channel, + /// so request handlers and resource-isolation classifiers cannot alter the evidence + /// later requests are checked against. when the + /// channel is not secured with a client certificate. + /// + public ByteString ClientChannelCertificate { get; } /// - /// The server certificate used to establsih the secure channel. + /// The DER encoded server certificate used to establish the secure channel. /// - public byte[]? ServerChannelCertificate { get; } + /// + /// Read-only, see . + /// + public ByteString ServerChannelCertificate { get; } /// /// The observed network address of the peer, when the transport exposes one. diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs index e0ff3022f8..b9ef57c6e2 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs @@ -104,7 +104,8 @@ protected string SecurityPolicyUri /// /// /// Certificate.RawData returns a new copy on every access; the listener needs the - /// bytes for every request it dispatches. + /// bytes for every request it dispatches. The array never leaves the stack: + /// exposes it as a read-only ByteString view. /// internal byte[]? ClientCertificateRawData => GetCachedRawData(ClientCertificate, ref m_clientCertificateRawData); diff --git a/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs b/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs index 0633acb167..fcd0627157 100644 --- a/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs +++ b/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs @@ -298,10 +298,7 @@ public static void HasAuthenticatedSecureChannel( public static ByteString GetClientCertificateFingerprint(ISystemContext context) { OperationContext operationContext = GetOperationContext(context); - byte[] clientCertificate = operationContext.ChannelContext?.ClientChannelCertificate ?? - throw new ServiceResultException( - StatusCodes.BadSecurityChecksFailed, - "Unable to bind the request to a SecureChannel client certificate."); + ByteString clientCertificate = operationContext.ChannelContext?.ClientChannelCertificate ?? default; if (clientCertificate.Length == 0) { @@ -311,10 +308,10 @@ public static ByteString GetClientCertificateFingerprint(ISystemContext context) } #if NET6_0_OR_GREATER - return ByteString.From(SHA256.HashData(clientCertificate)); + return ByteString.From(SHA256.HashData(clientCertificate.Span)); #else using SHA256 sha256 = SHA256.Create(); - return ByteString.From(sha256.ComputeHash(clientCertificate)); + return ByteString.From(sha256.ComputeHash(clientCertificate.ToArray())); #endif } diff --git a/src/Opc.Ua.Lds.Server/LdsServer.cs b/src/Opc.Ua.Lds.Server/LdsServer.cs index c2da324aaa..7713dc3c8d 100644 --- a/src/Opc.Ua.Lds.Server/LdsServer.cs +++ b/src/Opc.Ua.Lds.Server/LdsServer.cs @@ -493,8 +493,8 @@ protected virtual ServiceResult ValidateRegistration( new LocalizedText("ServerType is out of range.")); } - byte[]? certBytes = secureChannelContext?.ClientChannelCertificate; - if (certBytes == null || certBytes.Length == 0) + ByteString certBytes = secureChannelContext?.ClientChannelCertificate ?? default; + if (certBytes.Length == 0) { return new ServiceResult( StatusCodes.BadSecurityChecksFailed, @@ -503,7 +503,7 @@ protected virtual ServiceResult ValidateRegistration( try { - using var cert = Certificate.FromRawData(certBytes); + using var cert = Certificate.FromRawData(certBytes.Memory); IReadOnlyList applicationUris = X509Utils.GetApplicationUrisFromCertificate(cert); if (applicationUris.Count == 0) { diff --git a/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs b/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs index acf48853b4..d2692df51a 100644 --- a/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs +++ b/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs @@ -519,7 +519,7 @@ public override async ValueTask CloseSessionAsync( entry, endpoint.SecurityPolicyUri ?? string.Empty, endpoint.SecurityMode, - context.ChannelContext.ClientChannelCertificate.ToByteString()); + EmptyIfNull(context.ChannelContext.ClientChannelCertificate)); if (decision != RestoreDecision.Authorized) { m_logger.DistributedSessionRestoreRejected(TokenDigest(authenticationToken), decision); @@ -783,7 +783,7 @@ internal RestoreDecision AuthorizeRestore( } if (entry.OriginalClientChannelCertificate != - context.ChannelContext!.ClientChannelCertificate.ToByteString()) + context.ChannelContext!.ClientChannelCertificate) { return null; } @@ -912,7 +912,7 @@ private SharedSessionEntry BuildEntry( ClientCertificateChain = clientCertBlob, SecurityStateVersion = SharedSessionEntry.CurrentSecurityStateVersion, OriginalClientChannelCertificate = - context.ChannelContext!.ClientChannelCertificate.ToByteString(), + EmptyIfNull(context.ChannelContext!.ClientChannelCertificate), ClientUserId = null, ClientUserTokenType = UserTokenType.Anonymous, HasActivatedUserIdentity = false, @@ -1465,6 +1465,15 @@ private sealed record LivenessState(double SessionTimeout, DateTime MirroredCont private readonly ConcurrentDictionary m_tokensBySession = new(); private readonly ConcurrentDictionary m_mirrorLocks = new(); private readonly ConcurrentDictionary m_mirroredSequences = new(); + + /// + /// The channel certificate as the mirrored session state records it: empty, not + /// null, for a channel without a client certificate. + /// + private static ByteString EmptyIfNull(ByteString certificate) + { + return certificate.IsNull ? ByteString.Empty : certificate; + } } /// diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index 1055942831..85fb0babca 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -193,9 +193,10 @@ public ResourceIsolationOwner Classify( key = m_bindingKeys.GetValue(binding, value => new DerivedKey("user:" + Hash(Encoding.UTF8.GetBytes(value.ClientUserId!)))).Value; } - else if (secure && channelContext.ClientChannelCertificate is { Length: > 0 } certificate) + else if (secure && channelContext.ClientChannelCertificate.Length > 0) { - key = "application:" + Hash(certificate); + // classification-cache miss only + key = "application:" + Hash(channelContext.ClientChannelCertificate.ToArray()); } else { @@ -843,8 +844,9 @@ public ChannelSnapshot(SecureChannelContext context) m_channelId = context.SecureChannelId; SecurityPolicyUri = context.EndpointDescription?.SecurityPolicyUri; SecurityMode = context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid; + // ByteString is a read-only view, so the evidence cannot change after capture. m_certificate = SecurityPolicyUri != SecurityPolicies.None - ? context.ClientChannelCertificate?.AsSpan().ToArray() : null; + ? context.ClientChannelCertificate : default; m_peer = context.PeerAddress?.GetAddressBytes(); m_upstreamIdentity = context.UpstreamIdentity; } @@ -868,7 +870,7 @@ public bool Matches(SecureChannelContext context) SecurityPolicyUri == context.EndpointDescription?.SecurityPolicyUri && SecurityMode == (context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid) && (SecurityPolicyUri == SecurityPolicies.None || - m_certificate.AsSpan().SequenceEqual(context.ClientChannelCertificate.AsSpan())) && + m_certificate.Span.SequenceEqual(context.ClientChannelCertificate.Span)) && PeerMatches(context.PeerAddress) && ReferenceEquals(m_upstreamIdentity, context.UpstreamIdentity); } @@ -894,7 +896,7 @@ private bool PeerMatches(IPAddress? peer) } private readonly string m_channelId; - private readonly byte[]? m_certificate; + private readonly ByteString m_certificate; private readonly byte[]? m_peer; private readonly IUserIdentity? m_upstreamIdentity; } diff --git a/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs b/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs new file mode 100644 index 0000000000..f64496ff76 --- /dev/null +++ b/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs @@ -0,0 +1,48 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +namespace Opc.Ua.Server +{ + /// + /// Conversions of the read-only channel evidence in + /// for APIs that take arrays. + /// + internal static class ChannelEvidenceExtensions + { + /// + /// Returns a copy of the bytes, or null for a null byte string, matching + /// the nullable arrays the signature helpers expect. + /// + public static byte[]? ToArrayOrNull(this ByteString value) + { + return value.IsNull ? null : value.ToArray(); + } + } +} diff --git a/src/Opc.Ua.Server/Session/Session.cs b/src/Opc.Ua.Server/Session/Session.cs index 2ba782cdb3..c690876c7e 100644 --- a/src/Opc.Ua.Server/Session/Session.cs +++ b/src/Opc.Ua.Server/Session/Session.cs @@ -954,11 +954,11 @@ private void ValidateChannelBeforeActivate( byte[] clientNonceData = ClientNonce.ToArray(); byte[] dataToSign = securityPolicy.GetClientSignatureData( - context.ChannelContext.ChannelThumbprint, + context.ChannelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, m_serverCertificate.RawData, - context.ChannelContext.ServerChannelCertificate, - context.ChannelContext.ClientChannelCertificate, + context.ChannelContext.ServerChannelCertificate.ToArrayOrNull(), + context.ChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); if (!VerifyClientSignature(clientSignature!, dataToSign) && @@ -1010,11 +1010,11 @@ private bool VerifyClientSignatureOverServerCertificateChain( } byte[] dataToSign = securityPolicy.GetClientSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, [.. serverCertificateChainList], - channelContext.ServerChannelCertificate, - channelContext.ClientChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); return VerifyClientSignature(clientSignature, dataToSign); } @@ -1058,11 +1058,11 @@ private bool VerifyClientSignatureOverServerCertificate( byte[] serverCertificate) { byte[] dataToSign = securityPolicy.GetClientSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, serverCertificate, - channelContext.ServerChannelCertificate, - channelContext.ClientChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); return VerifyClientSignature(clientSignature, dataToSign); } @@ -1513,12 +1513,12 @@ await token.DecryptAsync( byte[] clientNonceData = ClientNonce.ToArray(); byte[] dataToSign = securityPolicy.GetUserTokenSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, m_serverCertificate.RawData, - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), ClientCertificate?.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData, EndpointDescription.SecurityMode); @@ -1545,12 +1545,12 @@ await token.DecryptAsync( } dataToSign = securityPolicy.GetUserTokenSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, [.. serverCertificateChainList], - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), ClientCertificate?.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData, EndpointDescription.SecurityMode); diff --git a/src/Opc.Ua.Server/Session/SessionManager.cs b/src/Opc.Ua.Server/Session/SessionManager.cs index e941aa68c0..9a7a5a3d4e 100644 --- a/src/Opc.Ua.Server/Session/SessionManager.cs +++ b/src/Opc.Ua.Server/Session/SessionManager.cs @@ -438,7 +438,10 @@ public virtual async ValueTask CreateSessionAsync( m_sessionActivationStates.Add( session, new SessionActivationState( - channelContext.ClientChannelCertificate.ToByteString(), + // empty rather than null for an unsecured channel, as before + channelContext.ClientChannelCertificate.IsNull + ? ByteString.Empty + : channelContext.ClientChannelCertificate, channelContext.EndpointDescription!.SecurityPolicyUri ?? SecurityPolicies.None, channelContext.EndpointDescription.SecurityMode)); @@ -748,7 +751,7 @@ private void ThrowIfClientNonceInUse(OperationContext context, ByteString client ((requiresClientCertificate && activationState.OriginalClientChannelCertificate.IsEmpty) || activationState.OriginalClientChannelCertificate != - channelContext.ClientChannelCertificate.ToByteString())) + channelContext.ClientChannelCertificate)) { throw new ServiceResultException( StatusCodes.BadSecurityChecksFailed, @@ -1286,21 +1289,23 @@ public bool TryGetSessionContext( SessionActivationState state; SessionBindingContext binding; ByteString originalClientChannelCertificate; - lock (m_bindingsLock) + + // Runs at admission and again at dispatch of every request. The session index + // and the activation-state table are concurrent collections, so the candidate + // snapshot is taken without the bindings lock; the check under the lock below + // confirms that the binding is still the committed one before it is returned. + if (authenticationToken.IsNull || + !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || + !m_sessionActivationStates.TryGetValue(currentSession, out SessionActivationState? currentState) || + currentState.IsCommitting || + currentState.BindingContext is not SessionBindingContext currentBinding) { - if (authenticationToken.IsNull || - !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || - !m_sessionActivationStates.TryGetValue(currentSession, out SessionActivationState? currentState) || - currentState.IsCommitting || - currentState.BindingContext is not SessionBindingContext currentBinding) - { - return false; - } - session = currentSession; - state = currentState; - binding = currentBinding; - originalClientChannelCertificate = state.OriginalClientChannelCertificate; + return false; } + session = currentSession; + state = currentState; + binding = currentBinding; + originalClientChannelCertificate = state.OriginalClientChannelCertificate; // Session diagnostics callbacks can query membership while holding a Session lock. // Probe Session state without the index lock, then check that the snapshot is still current. @@ -1313,7 +1318,7 @@ channelContext.EndpointDescription is not EndpointDescription endpoint || binding.SecurityMode != endpoint.SecurityMode || !string.Equals(binding.SecurityPolicyUri, endpoint.SecurityPolicyUri, StringComparison.Ordinal) || (binding.SecurityPolicyUri != SecurityPolicies.None && - originalClientChannelCertificate != channelContext.ClientChannelCertificate.ToByteString())) + originalClientChannelCertificate != channelContext.ClientChannelCertificate)) { return false; } @@ -1322,8 +1327,8 @@ channelContext.EndpointDescription is not EndpointDescription endpoint || { if (state.IsCommitting || !ReferenceEquals(state.BindingContext, binding) || - !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || - !ReferenceEquals(currentSession, session)) + !m_sessions.TryGetValue(authenticationToken, out ISession? committedSession) || + !ReferenceEquals(committedSession, session)) { return false; } @@ -1969,8 +1974,7 @@ private void ValidateAnonymousSessionlessChannel( endpoint.EndpointUrl != null && Utils.IsUriHttpsScheme(endpoint.EndpointUrl)); if (requiresClientCertificate && - (secureChannelContext.ClientChannelCertificate == null || - secureChannelContext.ClientChannelCertificate.Length == 0)) + secureChannelContext.ClientChannelCertificate.Length == 0) { throw ServiceResultException.Create( StatusCodes.BadSecurityChecksFailed, diff --git a/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs b/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs index 5b1efa1911..0c9660fc41 100644 --- a/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs +++ b/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs @@ -63,11 +63,11 @@ internal static class SessionSecurityPolicyHelper SecureChannelContext channelContext = context.ChannelContext!; byte[] dataToSign = securityPolicy!.GetServerSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), clientNonce.ToArray(), - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), parsedClientCertificate.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), serverNonce.ToArray()); return policies.CreateSignatureData( diff --git a/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs new file mode 100644 index 0000000000..28af03d742 --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs @@ -0,0 +1,89 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using NUnit.Framework; + +namespace Opc.Ua.Core.Tests.Stack.Server +{ + /// + /// Tests for the read-only channel evidence carried by . + /// + [TestFixture] + [Category("SecureChannelContext")] + [Parallelizable] + public class SecureChannelContextTests + { + /// + /// Missing evidence stays null rather than becoming empty: the session signature + /// helpers hash an empty certificate but skip a missing one. + /// + [Test] + public void MissingEvidenceIsNullNotEmpty() + { + var context = new SecureChannelContext("1", null, RequestEncoding.Binary); + + Assert.That(context.ClientChannelCertificate.IsNull, Is.True); + Assert.That(context.ServerChannelCertificate.IsNull, Is.True); + Assert.That(context.ChannelThumbprint.IsNull, Is.True); + } + + /// + /// Empty evidence stays empty and is not reported as null. + /// + [Test] + public void EmptyEvidenceIsNotNull() + { + var context = new SecureChannelContext("1", null, RequestEncoding.Binary, [], [], []); + + Assert.That(context.ClientChannelCertificate.IsNull, Is.False); + Assert.That(context.ClientChannelCertificate.IsEmpty, Is.True); + } + + /// + /// The evidence is a view of the transport's buffer, so contexts created for the + /// requests of one channel share it without copying it. + /// + [Test] + public void EvidenceWrapsTheTransportBufferWithoutCopying() + { + byte[] certificate = [1, 2, 3, 4]; + + var first = new SecureChannelContext("1", null, RequestEncoding.Binary, certificate); + var second = new SecureChannelContext("1", null, RequestEncoding.Binary, certificate); + + Assert.That(first.ClientChannelCertificate.Span.ToArray(), Is.EqualTo(certificate)); + Assert.That( + first.ClientChannelCertificate.Span.Overlaps(second.ClientChannelCertificate.Span), + Is.True); + Assert.That(first.ClientChannelCertificate, Is.EqualTo(second.ClientChannelCertificate)); + } + } +} diff --git a/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs b/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs index aa43a4f45e..0113cd8d36 100644 --- a/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs +++ b/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs @@ -112,7 +112,7 @@ public async Task SessionIsMirroredEncryptedOnActivateAndRemovedOnCloseAsync() Assert.That(entry.HasActivatedUserIdentity, Is.True); Assert.That( entry.OriginalClientChannelCertificate, - Is.EqualTo(context.ClientChannelCertificate.ToByteString())); + Is.EqualTo(context.ClientChannelCertificate)); // Encrypted at rest: a protector with a different key fails closed. using var wrongKey = new AesCbcHmacRecordProtector(MakeKey(2)); diff --git a/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs b/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs index 1b25d9e7e7..79c14cd08b 100644 --- a/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs +++ b/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs @@ -188,11 +188,11 @@ public static class ServerFixtureUtils createSessionResponse.ServerCertificate, server.MessageContext.Telemetry); byte[] dataToSign = securityPolicy.GetClientSignatureData( - secureChannelContext.ChannelThumbprint, + secureChannelContext.ChannelThumbprint.ToArrayOrNull(), createSessionResponse.ServerNonce.ToArray(), serverCertificateChain[0].RawData, - secureChannelContext.ServerChannelCertificate, - secureChannelContext.ClientChannelCertificate, + secureChannelContext.ServerChannelCertificate.ToArrayOrNull(), + secureChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonce.ToArray()); clientSignature = SecurityPolicies.Default.CreateSignatureData( securityPolicy, diff --git a/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs b/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs index 21b3131816..2968e11092 100644 --- a/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs +++ b/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs @@ -1015,7 +1015,7 @@ public void ClassifierCannotObserveCertificateOnNoneChannel() var classifier = new Mock(MockBehavior.Strict); ResourceIsolationIdentity identity = default; classifier.Setup(c => c.TryClassify( - It.Is(c => c.ClientChannelCertificate == null), + It.Is(c => c.ClientChannelCertificate.IsNull), null, out identity)).Returns(false); classifier.Setup(c => c.TryClassifyIngress(It.IsAny(), out identity)).Returns(false); using DefaultServerResourceIsolationProvider provider = CreateProvider(Options(), classifier.Object); diff --git a/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs b/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs index ea83026193..682189b25d 100644 --- a/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs @@ -582,8 +582,8 @@ public NonceHarness(Action? beforeSignature = null, bool forceRsa beforeSignature?.Invoke(this); ClientSignature = SecurityPolicies.Default.CreateSignatureData( policy, ClientCertificate, policy.GetClientSignatureData( - channel.ChannelThumbprint, ServerNonce.Data, ServerCertificate.RawData, - channel.ServerChannelCertificate, channel.ClientChannelCertificate, clientNonce.Data)); + channel.ChannelThumbprint.ToArrayOrNull(), ServerNonce.Data, ServerCertificate.RawData, + channel.ServerChannelCertificate.ToArrayOrNull(), channel.ClientChannelCertificate.ToArrayOrNull(), clientNonce.Data)); } catch { diff --git a/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs b/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs index 8f0152e416..04dcf1dd8d 100644 --- a/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs +++ b/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs @@ -654,7 +654,7 @@ private async ValueTask CreateRestoredSessionAsync( context.ChannelContext!.EndpointDescription!; SetRestoredSessionTransferSecurityState( session, - context.ChannelContext.ClientChannelCertificate.ToByteString(), + context.ChannelContext.ClientChannelCertificate, endpoint.SecurityPolicyUri ?? SecurityPolicies.None, endpoint.SecurityMode, m_behavior == RestoreBehavior.RestoreWithDifferentOwner diff --git a/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs b/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs index 3577087b24..00f73bc469 100644 --- a/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs +++ b/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs @@ -528,12 +528,12 @@ public async Task CertificateUserPolicyVerifiesProofOnNoneChannelAsync(string pr SecurityPolicyInfo policy = SecurityPolicies.Default.GetInfo(SecurityPolicies.Basic256Sha256)!; SecureChannelContext channel = created.Context.ChannelContext!; byte[] dataToSign = policy.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray()); SignatureData userSignature = proof switch { @@ -607,21 +607,21 @@ public async Task EnhancedCertificateUserPolicyUsesNoneVariantOnNoneChannelAsync SecureChannelContext channel = created.Context.ChannelContext!; byte[] dataToSign = noneVariant ? policy!.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray(), MessageSecurityMode.None) : policy!.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray()); SignatureData userSignature = await SecurityPolicies.Default.CreateSignatureDataAsync( policy, @@ -1459,11 +1459,11 @@ private SignatureData CreateClientSignature( SecurityPolicyInfo securityPolicy = SecurityPolicies.Default.GetInfo( context.ChannelContext!.EndpointDescription!.SecurityPolicyUri!)!; byte[] dataToSign = securityPolicy.GetClientSignatureData( - context.ChannelContext.ChannelThumbprint, + context.ChannelContext.ChannelThumbprint.ToArrayOrNull(), serverNonce.ToArray(), m_serverCertificate.RawData, - context.ChannelContext.ServerChannelCertificate, - context.ChannelContext.ClientChannelCertificate, + context.ChannelContext.ServerChannelCertificate.ToArrayOrNull(), + context.ChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonce.ToArray()); return SecurityPolicies.Default.CreateSignatureData( securityPolicy, From a0d9b9e7a42442c3e3cf32a8ae70ff06eab5ff6d Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:08 +0200 Subject: [PATCH 05/16] Tests: cover ObjectPool, the per-request system context cache and the translation snapshot ObjectPool: reuse, bounded retention, the per-processor cap, no shared rental under concurrent Get/Return, and no lost capacity after concurrent use. OperationContext.GetSystemContext: one copy per template, never another node manager's copy when templates alternate, and concurrent lookups bound to the same operation. ResourceManager: updates and new locales are visible after a snapshot was built, and concurrent readers see only complete translations. Co-Authored-By: Claude Opus 5.5 --- .../Types/Utils/ObjectPoolTests.cs | 196 ++++++++++++++++++ ...OperationContextSystemContextCacheTests.cs | 140 +++++++++++++ .../ResourceManagerSnapshotTests.cs | 134 ++++++++++++ 3 files changed, 470 insertions(+) create mode 100644 tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs create mode 100644 tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs create mode 100644 tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs diff --git a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs new file mode 100644 index 0000000000..66584127de --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs @@ -0,0 +1,196 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace Opc.Ua.Core.Tests.Types.UtilsTests +{ + /// + /// Tests for the bounded lock-free . + /// + [TestFixture] + [Category("Utils")] + [Parallelizable] + public class ObjectPoolTests + { + /// + /// A returned object is handed out again before a new one is created. + /// + [Test] + public void ReturnedObjectIsReused() + { + int created = 0; + var pool = new ObjectPool(() => { created++; return new object(); }, 4); + + object first = pool.Get(); + pool.Return(first); + object second = pool.Get(); + + Assert.That(second, Is.SameAs(first)); + Assert.That(created, Is.EqualTo(1)); + } + + /// + /// The pool retains at most its capacity; objects returned beyond it are dropped. + /// + [Test] + public void RetainsAtMostTheCapacity() + { + int capacity = Math.Min(3, Environment.ProcessorCount * 4); + var pool = new ObjectPool(() => new object(), capacity); + var rented = new List(); + for (int ii = 0; ii < capacity + 5; ii++) + { + rented.Add(pool.Get()); + } + foreach (object item in rented) + { + pool.Return(item); + } + + var reused = new HashSet(); + for (int ii = 0; ii < capacity + 5; ii++) + { + object item = pool.Get(); + if (rented.Contains(item)) + { + reused.Add(item); + } + } + + Assert.That(reused, Has.Count.EqualTo(capacity)); + } + + /// + /// The capacity is capped at four objects per processor. + /// + [Test] + public void CapacityIsCappedByProcessorCount() + { + int cap = Environment.ProcessorCount * 4; + var pool = new ObjectPool(() => new object(), cap * 10); + var rented = new List(); + for (int ii = 0; ii < cap * 2; ii++) + { + rented.Add(pool.Get()); + } + foreach (object item in rented) + { + pool.Return(item); + } + + int reused = 0; + for (int ii = 0; ii < cap * 2; ii++) + { + if (rented.Contains(pool.Get())) + { + reused++; + } + } + + Assert.That(reused, Is.EqualTo(cap)); + } + + /// + /// Under concurrent Get and Return an object is never held by two renters at once. + /// + [Test] + public void ConcurrentRentalsNeverShareAnObject() + { + var pool = new ObjectPool(() => new Rental(), 64); + int duplicates = 0; + + Parallel.For(0, Environment.ProcessorCount * 2, _ => + { + for (int ii = 0; ii < 20000; ii++) + { + Rental rental = pool.Get(); + if (Interlocked.Exchange(ref rental.InUse, 1) != 0) + { + Interlocked.Increment(ref duplicates); + } + Thread.SpinWait(ii % 8); + Volatile.Write(ref rental.InUse, 0); + pool.Return(rental); + } + }); + + Assert.That(duplicates, Is.Zero); + } + + /// + /// Concurrent use does not lose capacity: once quiet, the pool still serves its retained + /// objects without creating new ones. + /// + [Test] + public void ConcurrentUseDoesNotLoseCapacity() + { + int created = 0; + var pool = new ObjectPool( + () => + { + Interlocked.Increment(ref created); + return new object(); + }, + Environment.ProcessorCount * 4); + var seen = new ConcurrentDictionary(); + + Parallel.For(0, Environment.ProcessorCount, _ => + { + for (int ii = 0; ii < 10000; ii++) + { + object item = pool.Get(); + seen.TryAdd(item, true); + pool.Return(item); + } + }); + + int createdAfterLoad = Volatile.Read(ref created); + var rented = new List(); + for (int ii = 0; ii < Math.Min(createdAfterLoad, Environment.ProcessorCount * 4); ii++) + { + rented.Add(pool.Get()); + } + + Assert.That(Volatile.Read(ref created), Is.EqualTo(createdAfterLoad)); + Assert.That(rented, Is.All.Matches(seen.ContainsKey)); + } + + private sealed class Rental + { + public int InUse; + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs b/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs new file mode 100644 index 0000000000..01ab911217 --- /dev/null +++ b/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs @@ -0,0 +1,140 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System.Linq; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests +{ + /// + /// Tests for the per-request system context copy cached on , + /// which node managers reuse for the per-node callbacks of one request. + /// + [TestFixture] + [Category("OperationContext")] + [SetCulture("en-us")] + [SetUICulture("en-us")] + [Parallelizable] + public class OperationContextSystemContextCacheTests + { + /// + /// Repeated lookups with the same template return the same copy, bound to the operation. + /// + [Test] + public void SameTemplateReturnsOneCopyBoundToTheOperation() + { + ServerSystemContext template = CreateTemplate(); + using OperationContext operation = CreateOperation(); + + ServerSystemContext first = operation.GetSystemContext(template); + ServerSystemContext second = operation.GetSystemContext(template); + + Assert.That(second, Is.SameAs(first)); + Assert.That(first, Is.Not.SameAs(template)); + Assert.That(first.OperationContext, Is.SameAs(operation)); + } + + /// + /// A copy made from one node manager's template is never handed to another node manager. + /// + [Test] + public void SwitchingTemplatesNeverReturnsAnotherManagersCopy() + { + ServerSystemContext templateA = CreateTemplate(systemHandle: "A"); + ServerSystemContext templateB = CreateTemplate(systemHandle: "B"); + using OperationContext operation = CreateOperation(); + + ServerSystemContext a1 = operation.GetSystemContext(templateA); + ServerSystemContext b1 = operation.GetSystemContext(templateB); + ServerSystemContext a2 = operation.GetSystemContext(templateA); + + Assert.That(a1.SystemHandle, Is.EqualTo("A")); + Assert.That(b1.SystemHandle, Is.EqualTo("B")); + Assert.That(a2.SystemHandle, Is.EqualTo("A")); + Assert.That(b1, Is.Not.SameAs(a1)); + Assert.That(b1.OperationContext, Is.SameAs(operation)); + Assert.That(a2.OperationContext, Is.SameAs(operation)); + } + + /// + /// Concurrent lookups with alternating templates always get a copy of the template they + /// passed, bound to the same operation. + /// + [Test] + public void ConcurrentLookupsAlwaysReturnACopyOfTheRequestedTemplate() + { + ServerSystemContext[] templates = + [ + CreateTemplate(systemHandle: "A"), + CreateTemplate(systemHandle: "B"), + CreateTemplate(systemHandle: "C") + ]; + using OperationContext operation = CreateOperation(); + + int mismatches = 0; + Parallel.For(0, 30000, ii => + { + ServerSystemContext template = templates[ii % templates.Length]; + ServerSystemContext copy = operation.GetSystemContext(template); + if (!Equals(copy.SystemHandle, template.SystemHandle) || + !ReferenceEquals(copy.OperationContext, operation)) + { + System.Threading.Interlocked.Increment(ref mismatches); + } + }); + + Assert.That(mismatches, Is.Zero); + Assert.That( + templates.Select(t => operation.GetSystemContext(t).SystemHandle), + Is.EqualTo(new object[] { "A", "B", "C" })); + } + + private static ServerSystemContext CreateTemplate(object? systemHandle = null) + { + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(NUnitTelemetryContext.Create()); + server.Setup(s => s.NamespaceUris).Returns(new NamespaceTable()); + server.Setup(s => s.ServerUris).Returns(new StringTable()); + return new ServerSystemContext(server.Object) { SystemHandle = systemHandle }; + } + + private static OperationContext CreateOperation() + { + return new OperationContext( + new RequestHeader(), + null, + RequestType.Read, + RequestLifetime.None); + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs new file mode 100644 index 0000000000..7b0d53add2 --- /dev/null +++ b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs @@ -0,0 +1,134 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests +{ + /// + /// Tests that the lock-free translation snapshot of is rebuilt + /// after every change, so translations always reflect the tables as written. + /// + [TestFixture] + [Category("ResourceManager")] + [Parallelizable] + [SetCulture("en-us")] + [SetUICulture("en-us")] + public class ResourceManagerSnapshotTests + { + /// + /// A translation read after an update, and after a new locale is added, sees the change. + /// + [Test] + public void UpdatesAndNewLocalesAreVisibleAfterASnapshotWasBuilt() + { + using ResourceManager resourceManager = CreateResourceManager(); + resourceManager.Add("greeting", "de-DE", "Hallo"); + + // builds the snapshot + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Hallo")); + + resourceManager.Add("greeting", "de-DE", "Servus"); + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Servus")); + + resourceManager.Add("greeting", "fr-FR", "Bonjour"); + Assert.That(Translate(resourceManager, "fr-FR"), Is.EqualTo("Bonjour")); + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Servus")); + Assert.That(resourceManager.GetAvailableLocales(), Does.Contain("fr-FR")); + } + + /// + /// Readers running while translations are added never fail, never see a partially + /// written table, and see every translation once the writer is done. + /// + [Test] + public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() + { + using ResourceManager resourceManager = CreateResourceManager(); + const int keyCount = 500; + using var cts = new CancellationTokenSource(); + int wrongTexts = 0; + + Task[] readers = new Task[4]; + for (int r = 0; r < readers.Length; r++) + { + readers[r] = Task.Run(() => + { + while (!cts.IsCancellationRequested) + { + for (int ii = 0; ii < keyCount; ii += 37) + { + string key = "key" + ii.ToString(CultureInfo.InvariantCulture); + LocalizedText text = resourceManager.Translate(["de-DE"], key, "default"); + // either not added yet (default) or the complete translation + if (text.Text != "default" && text.Text != "Wert" + ii.ToString(CultureInfo.InvariantCulture)) + { + Interlocked.Increment(ref wrongTexts); + } + } + } + }); + } + + for (int ii = 0; ii < keyCount; ii++) + { + resourceManager.Add( + "key" + ii.ToString(CultureInfo.InvariantCulture), + "de-DE", + "Wert" + ii.ToString(CultureInfo.InvariantCulture)); + } + cts.Cancel(); + Assert.That(Task.WaitAll(readers, 30000), Is.True); + + Assert.That(wrongTexts, Is.Zero); + for (int ii = 0; ii < keyCount; ii++) + { + string key = "key" + ii.ToString(CultureInfo.InvariantCulture); + Assert.That( + resourceManager.Translate(["de-DE"], key, "default").Text, + Is.EqualTo("Wert" + ii.ToString(CultureInfo.InvariantCulture))); + } + } + + private static ResourceManager CreateResourceManager() + { + return new ResourceManager(new ApplicationConfiguration(NUnitTelemetryContext.Create())); + } + + private static string? Translate(ResourceManager resourceManager, string locale) + { + return resourceManager.Translate([locale], "greeting", "Hello").Text; + } + } +} From 840b264142ec56463e9e42f601b6b25000d58bad Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:09 +0200 Subject: [PATCH 06/16] perf: follow repository conventions in the harness and trace analyzer Add the OPC Foundation license header, drop the per-project Nullable, warnings-as-errors and analysis-level overrides and fix the diagnostics instead, remove comment section dividers, and record latency samples in a ConcurrentBag instead of locking List instances. The only suppression left is a documented CA2000 where the session takes ownership of a subscription. Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/Program.cs | 1144 ++++++++++------- .../ServerLoadHarness.csproj | 11 +- perf/TraceAnalyzer/Program.cs | 384 ++++-- perf/TraceAnalyzer/TraceAnalyzer.csproj | 6 - 4 files changed, 906 insertions(+), 639 deletions(-) diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index a1bc5d8c73..320d3807ba 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -1,647 +1,821 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + // Server load / profiling harness. Not part of the product; see perf/README.md. // Modes: // server --port P [--diag 0|1] [--audit 0|1] [--threads N] // Starts a ReferenceServer and reads commands from stdin: mark | report | quit // run --scenario read|write|sub|browse [--sessions N] [--nodes K] [--inflight I] // [--duration S] [--warmup S] [--security none|sign|encrypt] [--diag 0|1] [--audit 0|1] -// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] +// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] [--out file] // Spawns the server child, drives the load and prints a report of client latency // and server-side CPU / GC / allocation deltas for the measurement window. +using System; +using System.Collections; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; +using System.IO; +using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Opc.Ua; using Opc.Ua.Client; using Opc.Ua.Client.TestFramework; using Opc.Ua.Configuration; using Opc.Ua.Server.TestFramework; using Quickstarts.ReferenceServer; -internal static class Program +namespace Opc.Ua.Perf.ServerLoadHarness { - private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); - - private static string Arg(string name, string def) => s_args.TryGetValue(name, out string? v) ? v : def; - private static int ArgInt(string name, int def) => int.Parse(Arg(name, def.ToString(CultureInfo.InvariantCulture)), CultureInfo.InvariantCulture); - - public static async Task Main(string[] args) + /// + /// Entry point of the harness. + /// + public static class Program { - for (int i = 1; i < args.Length; i++) + private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Runs the server or the load driver. + /// + public static async Task Main(string[] args) { - if (args[i].StartsWith("--", StringComparison.Ordinal)) + for (int i = 1; i < args.Length; i++) { - string key = args[i][2..]; - string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) ? args[++i] : "1"; - s_args[key] = value; + if (args[i].StartsWith("--", StringComparison.Ordinal)) + { + string key = args[i][2..]; + string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) + ? args[++i] + : "1"; + s_args[key] = value; + } } + string mode = args.Length > 0 ? args[0] : "run"; + return mode == "server" + ? await RunServerAsync().ConfigureAwait(false) + : await RunDriverAsync(args).ConfigureAwait(false); } - string mode = args.Length > 0 ? args[0] : "run"; - return mode switch - { - "server" => await RunServerAsync().ConfigureAwait(false), - _ => await RunDriverAsync(args).ConfigureAwait(false) - }; - } - - // ------------------------------------------------------------------ server - - private static ITelemetryContext CreateTelemetry(LogLevel level) => - DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); - private static async Task RunServerAsync() - { - int port = ArgInt("port", 62555); - bool diag = ArgInt("diag", 0) != 0; - bool audit = ArgInt("audit", 0) != 0; - ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); - - string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); - var application = new ApplicationInstance(telemetry) - { - ApplicationName = "PerfReferenceServer", - ApplicationType = ApplicationType.Server - }; - string endpointUrl = $"opc.tcp://localhost:{port}/PerfReferenceServer"; - ArrayOf certs = ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( - "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", CertificateStoreType.Directory, pkiRoot); - ApplicationConfiguration config = await application - .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") - .SetMaxByteStringLength(4 * 1024 * 1024) - .SetMaxArrayLength(1024 * 1024) - .SetMaxMessageSize(16 * 1024 * 1024) - .AsServer([endpointUrl]) - .AddUnsecurePolicyNone() - .AddSignPolicies() - .AddSignAndEncryptPolicies() - .SetMaxChannelCount(2100) - .SetDiagnosticsEnabled(diag) - .SetAuditingEnabled(audit) - .SetShutdownDelay(0) - .AddSecurityConfiguration(certs, pkiRoot) - .SetAutoAcceptUntrustedCertificates(true) - .CreateAsync().ConfigureAwait(false); - config.ServerConfiguration!.MaxSessionCount = 2000; - config.ServerConfiguration.MaxSubscriptionCount = 20000; - config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; - config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); - config.ServerConfiguration.MinRequestThreadCount = 50; - - if (!await application.CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime).ConfigureAwait(false)) + internal static string Arg(string name, string defaultValue) { - Console.WriteLine("ERROR cert"); - return 1; + return s_args.TryGetValue(name, out string? value) ? value : defaultValue; } - var server = new ReferenceServer(telemetry); - server.TransportBindings = TestTransportBindings.WithAllSchemes(); - await application.StartAsync(server).ConfigureAwait(false); - Console.WriteLine($"READY {endpointUrl} {Environment.ProcessId}"); + internal static int ArgInt(string name, int defaultValue) + { + return int.Parse( + Arg(name, defaultValue.ToString(CultureInfo.InvariantCulture)), + CultureInfo.InvariantCulture); + } - ServerStats mark = ServerStats.Capture(); - string? line; - while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) + internal static List GetNodes(ISession session, int count, Type? onlyType = null) { - switch (line.Trim()) - { - case "mark": - GC.Collect(); - GC.WaitForPendingFinalizers(); - mark = ServerStats.Capture(); - Console.WriteLine("MARKED"); - break; - case "report": - Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); - break; - case "quit": - await server.StopAsync().ConfigureAwait(false); - return 0; + var result = new List(); + foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + { + if (onlyType != null && type != onlyType) + { + continue; + } + foreach (ExpandedNodeId id in ids) + { + if (result.Count == count) + { + return result; + } + result.Add(ExpandedNodeId.ToNodeId(id, session.NamespaceUris)); + } } + return result; } - return 0; - } - private sealed record ServerStats(long Alloc, int G0, int G1, int G2, TimeSpan Pause, TimeSpan Cpu, long Contention, long Stamp, int Threads, long WorkingSet, long Heap) - { - public static ServerStats Capture() + private static ITelemetryContext CreateTelemetry(LogLevel level) { - using var p = Process.GetCurrentProcess(); - return new ServerStats( - GC.GetTotalAllocatedBytes(true), GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), - GC.GetTotalPauseDuration(), p.TotalProcessorTime, Monitor.LockContentionCount, Stopwatch.GetTimestamp(), - ThreadPool.ThreadCount, p.WorkingSet64, GC.GetGCMemoryInfo().HeapSizeBytes); + return DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); } - public string Delta(ServerStats m) + private static async Task RunServerAsync() { - double sec = Stopwatch.GetElapsedTime(m.Stamp, Stamp).TotalSeconds; - return string.Create(CultureInfo.InvariantCulture, - $"sec={sec:F2} allocMB={(Alloc - m.Alloc) / 1048576.0:F1} gen0={G0 - m.G0} gen1={G1 - m.G1} gen2={G2 - m.G2} pauseMs={(Pause - m.Pause).TotalMilliseconds:F0} cpuMs={(Cpu - m.Cpu).TotalMilliseconds:F0} cpuCores={(Cpu - m.Cpu).TotalSeconds / sec:F2} contention={Contention - m.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + var application = new ApplicationInstance(telemetry) + { + ApplicationName = "PerfReferenceServer", + ApplicationType = ApplicationType.Server + }; + try + { + return await RunServerAsync(application, telemetry).ConfigureAwait(false); + } + finally + { + await application.DisposeAsync().ConfigureAwait(false); + } } - } - // ------------------------------------------------------------------ driver + private static async Task RunServerAsync(ApplicationInstance application, ITelemetryContext telemetry) + { + int port = ArgInt("port", 62555); + bool diagnostics = ArgInt("diag", 0) != 0; + bool auditing = ArgInt("audit", 0) != 0; + string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); + string endpointUrl = FormattableString.Invariant($"opc.tcp://localhost:{port}/PerfReferenceServer"); + ArrayOf certificates = + ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( + "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", + CertificateStoreType.Directory, + pkiRoot); + ApplicationConfiguration config = await application + .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") + .SetMaxByteStringLength(4 * 1024 * 1024) + .SetMaxArrayLength(1024 * 1024) + .SetMaxMessageSize(16 * 1024 * 1024) + .AsServer([endpointUrl]) + .AddUnsecurePolicyNone() + .AddSignPolicies() + .AddSignAndEncryptPolicies() + .SetMaxChannelCount(2100) + .SetDiagnosticsEnabled(diagnostics) + .SetAuditingEnabled(auditing) + .SetShutdownDelay(0) + .AddSecurityConfiguration(certificates, pkiRoot) + .SetAutoAcceptUntrustedCertificates(true) + .CreateAsync() + .ConfigureAwait(false); + config.ServerConfiguration!.MaxSessionCount = 2000; + config.ServerConfiguration.MaxSubscriptionCount = 20000; + config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; + config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); + config.ServerConfiguration.MinRequestThreadCount = 50; + + if (!await application + .CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime) + .ConfigureAwait(false)) + { + Console.WriteLine("ERROR cert"); + return 1; + } - private static async Task RunDriverAsync(string[] args) - { - string scenario = Arg("scenario", "read"); - int sessionsCount = ArgInt("sessions", 20); - int durationSec = ArgInt("duration", 20); - int warmupSec = ArgInt("warmup", 10); - int port = ArgInt("port", 62555 + Environment.ProcessId % 1000); - string security = Arg("security", "none"); - - string exe = Environment.ProcessPath!; - var psi = new ProcessStartInfo(exe, $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}") - { - RedirectStandardInput = true, - RedirectStandardOutput = true, - UseShellExecute = false - }; - foreach (System.Collections.DictionaryEntry kv in Environment.GetEnvironmentVariables()) - { - string k = (string)kv.Key; - if (k.StartsWith("SERVER_", StringComparison.Ordinal)) + using var server = new ReferenceServer(telemetry) + { + TransportBindings = TestTransportBindings.WithAllSchemes() + }; + await application.StartAsync(server).ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant($"READY {endpointUrl} {Environment.ProcessId}")); + + ServerStats mark = ServerStats.Capture(); + string? line; + while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) { - psi.Environment[k[7..]] = (string?)kv.Value; + switch (line.Trim()) + { + case "mark": + GC.Collect(); + GC.WaitForPendingFinalizers(); + mark = ServerStats.Capture(); + Console.WriteLine("MARKED"); + break; + case "report": + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + break; + case "quit": + await server.StopAsync().ConfigureAwait(false); + return 0; + } } + return 0; } - using Process server = Process.Start(psi)!; - var serverLines = new BlockingCollection(); - _ = Task.Run(() => + + private static async Task RunDriverAsync(string[] args) { - string? l; - while ((l = server.StandardOutput.ReadLine()) != null) + string scenario = Arg("scenario", "read"); + int sessionCount = ArgInt("sessions", 20); + int durationSeconds = ArgInt("duration", 20); + int warmupSeconds = ArgInt("warmup", 10); + int port = ArgInt("port", 62555 + (Environment.ProcessId % 1000)); + string security = Arg("security", "none"); + + var startInfo = new ProcessStartInfo( + Environment.ProcessPath!, + FormattableString.Invariant( + $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}")) { - if (l.StartsWith("READY", StringComparison.Ordinal) || l.StartsWith("REPORT", StringComparison.Ordinal) || l.StartsWith("MARKED", StringComparison.Ordinal) || l.StartsWith("ERROR", StringComparison.Ordinal)) + RedirectStandardInput = true, + RedirectStandardOutput = true, + UseShellExecute = false + }; + foreach (DictionaryEntry variable in Environment.GetEnvironmentVariables()) + { + string name = (string)variable.Key; + if (name.StartsWith("SERVER_", StringComparison.Ordinal)) { - serverLines.Add(l); + startInfo.Environment[name[7..]] = (string?)variable.Value; } - else if (s_args.ContainsKey("verbose")) + } + + using Process server = Process.Start(startInfo)!; + using var serverLines = new BlockingCollection(); + _ = Task.Run(() => PumpServerOutput(server, serverLines)); + string ready = serverLines.Take(); + if (!ready.StartsWith("READY", StringComparison.Ordinal)) + { + Console.WriteLine(ready); + return 1; + } + string url = ready.Split(' ')[1]; + Console.WriteLine(FormattableString.Invariant($"server pid={server.Id} url={url}")); + + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + using var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; + await client + .LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient") + .ConfigureAwait(false); + string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; + MessageSecurityMode mode = security switch + { + "sign" => MessageSecurityMode.Sign, + "encrypt" => MessageSecurityMode.SignAndEncrypt, + _ => MessageSecurityMode.None + }; + ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); + var selected = new List(); + foreach (EndpointDescription endpoint in endpoints) + { + if (endpoint.SecurityPolicyUri == policy && endpoint.SecurityMode == mode) { - Console.WriteLine("[server] " + l); + selected.Add(endpoint); } } - serverLines.CompleteAdding(); - }); - string ready = serverLines.Take(); - if (!ready.StartsWith("READY", StringComparison.Ordinal)) - { - Console.WriteLine(ready); - return 1; + + var sessions = new ISession[sessionCount]; + var connectTime = Stopwatch.StartNew(); + await Parallel.ForAsync( + 0, + sessionCount, + new ParallelOptions { MaxDegreeOfParallelism = 32 }, + async (i, ct) => sessions[i] = await client + .ConnectAsync(new Uri(url), policy, selected) + .ConfigureAwait(false)) + .ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"connected {sessionCount} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); + + Workload workload = scenario == "sub" + ? new SubscribeWorkload( + sessions, + ArgInt("nodes", 100), + ArgInt("subs", 5), + ArgInt("pub", 100), + ArgInt("write-interval", 100)) + : new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)); + await workload.SetupAsync().ConfigureAwait(false); + + using var stop = new CancellationTokenSource(); + Task run = workload.RunAsync(stop.Token); + await Task.Delay(TimeSpan.FromSeconds(warmupSeconds)).ConfigureAwait(false); + + using Process? tracer = StartTracer(server.Id, durationSeconds, out string? trace); + if (tracer != null) + { + await Task.Delay(2000).ConfigureAwait(false); + } + + await server.StandardInput.WriteLineAsync("mark").ConfigureAwait(false); + Take(serverLines, "MARKED"); + workload.ResetStats(); + var window = Stopwatch.StartNew(); + await Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ConfigureAwait(false); + await server.StandardInput.WriteLineAsync("report").ConfigureAwait(false); + string report = Take(serverLines, "REPORT")[7..]; + string clientReport = workload.Report(window.Elapsed.TotalSeconds); + await stop.CancelAsync().ConfigureAwait(false); + try + { + await run.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // the workload stops by cancellation + } + + if (tracer != null) + { + await tracer.WaitForExitAsync().ConfigureAwait(false); + Console.WriteLine("trace: " + trace); + } + + var result = new StringBuilder(); + result.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionCount} ") + .Append(string.Join(' ', args.Skip(1))) + .Append('\n') + .Append(" client: ").Append(clientReport).Append('\n') + .Append(" server: ").Append(report).Append('\n'); + double operations = workload.Operations; + if (operations > 0) + { + (double allocatedMb, double cpuMs) = ParseServerReport(report); + result.Append( + CultureInfo.InvariantCulture, + $" per-op: allocB={allocatedMb * 1048576 / operations:F0} cpuUs={cpuMs * 1000 / operations:F1} ({workload.OperationName})\n"); + } + Console.Write(result.ToString()); + if (s_args.TryGetValue("out", out string? outFile)) + { + await File.AppendAllTextAsync(outFile, result.ToString()).ConfigureAwait(false); + } + + await Parallel.ForEachAsync(sessions, async (session, ct) => + { + try + { + await session.CloseAsync(ct).ConfigureAwait(false); + } + catch (ServiceResultException) + { + // the server may already be shutting the session down + } + session.Dispose(); + }).ConfigureAwait(false); + await server.StandardInput.WriteLineAsync("quit").ConfigureAwait(false); + if (!server.WaitForExit(10000)) + { + server.Kill(); + } + return 0; } - string url = ready.Split(' ')[1]; - Console.WriteLine($"server pid={server.Id} url={url}"); - - ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); - var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; - await client.LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient").ConfigureAwait(false); - string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; - ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); - MessageSecurityMode wantedMode = security switch - { - "sign" => MessageSecurityMode.Sign, - "encrypt" => MessageSecurityMode.SignAndEncrypt, - _ => MessageSecurityMode.None - }; - var filtered = new List(); - foreach (EndpointDescription ep in endpoints) + + private static void PumpServerOutput(Process server, BlockingCollection lines) { - if (ep.SecurityPolicyUri == policy && ep.SecurityMode == wantedMode) + string? line; + while ((line = server.StandardOutput.ReadLine()) != null) { - filtered.Add(ep); + if (line.StartsWith("READY", StringComparison.Ordinal) || + line.StartsWith("REPORT", StringComparison.Ordinal) || + line.StartsWith("MARKED", StringComparison.Ordinal) || + line.StartsWith("ERROR", StringComparison.Ordinal)) + { + lines.Add(line); + } + else if (s_args.ContainsKey("verbose")) + { + Console.WriteLine("[server] " + line); + } } + lines.CompleteAdding(); } - var sessions = new ISession[sessionsCount]; - var connectSw = Stopwatch.StartNew(); - await Parallel.ForAsync(0, sessionsCount, new ParallelOptions { MaxDegreeOfParallelism = 32 }, async (i, ct) => - sessions[i] = await client.ConnectAsync(new Uri(url), policy, filtered).ConfigureAwait(false)).ConfigureAwait(false); - Console.WriteLine($"connected {sessionsCount} sessions ({security}) in {connectSw.ElapsedMilliseconds} ms"); - - Workload workload = scenario switch + private static Process? StartTracer(int processId, int durationSeconds, out string? trace) { - "sub" => new SubscribeWorkload(sessions, ArgInt("nodes", 100), ArgInt("subs", 5), ArgInt("pub", 100), ArgInt("write-interval", 100)), - _ => new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)) - }; - await workload.SetupAsync().ConfigureAwait(false); - - using var stop = new CancellationTokenSource(); - Task run = workload.RunAsync(stop.Token); - await Task.Delay(TimeSpan.FromSeconds(warmupSec)).ConfigureAwait(false); - - Process? tracer = null; - string? trace = s_args.TryGetValue("trace", out string? t) ? t : null; - if (trace != null) - { - string profile = Arg("trace-profile", "cpu"); - string providers = profile switch + if (!s_args.TryGetValue("trace", out trace)) + { + return null; + } + string providers = Arg("trace-profile", "cpu") switch { - // allocation sampling (GCAllocationTick ~ every 100KB) + GC events - "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5 --clrevents gc --clreventlevel verbose", + // allocation sampling (GCAllocationTick, about every 100 KB) and GC events + "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5", "gc" => "--profile gc-verbose", + // the Contention keyword "contention" => "--providers Microsoft-Windows-DotNETRuntime:0x4000:4", _ => "--profile dotnet-sampled-thread-time" }; - tracer = Process.Start(new ProcessStartInfo("dotnet-trace", $"collect --process-id {server.Id} {providers} -o \"{trace}\" --duration 00:00:{durationSec:D2}") + return Process.Start(new ProcessStartInfo( + "dotnet-trace", + FormattableString.Invariant( + $"collect --process-id {processId} {providers} -o \"{trace}\" --duration 00:00:{durationSeconds:D2}")) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }); - await Task.Delay(2000).ConfigureAwait(false); } - server.StandardInput.WriteLine("mark"); - Take(serverLines, "MARKED"); - workload.ResetStats(); - var sw = Stopwatch.StartNew(); - await Task.Delay(TimeSpan.FromSeconds(durationSec)).ConfigureAwait(false); - server.StandardInput.WriteLine("report"); - string report = Take(serverLines, "REPORT"); - double elapsed = sw.Elapsed.TotalSeconds; - string clientReport = workload.Report(elapsed); - stop.Cancel(); - try + private static string Take(BlockingCollection lines, string prefix) { - await run.ConfigureAwait(false); + while (true) + { + string line = lines.Take(); + if (line.StartsWith(prefix, StringComparison.Ordinal)) + { + return line; + } + } } - catch (OperationCanceledException) + + private static (double AllocatedMb, double CpuMs) ParseServerReport(string report) { + Dictionary values = report + .Split(' ') + .Select(p => p.Split('=')) + .Where(p => p.Length == 2) + .ToDictionary(p => p[0], p => p[1], StringComparer.Ordinal); + return ( + double.Parse(values["allocMB"], CultureInfo.InvariantCulture), + double.Parse(values["cpuMs"], CultureInfo.InvariantCulture)); } + } - if (tracer != null) + /// + /// Process-wide counters of the server process, captured at the start and end of the window. + /// + internal sealed record ServerStats( + long Allocated, + int Gen0, + int Gen1, + int Gen2, + TimeSpan Pause, + TimeSpan Cpu, + long Contention, + long Timestamp, + int Threads, + long WorkingSet, + long Heap) + { + public static ServerStats Capture() { - await tracer.WaitForExitAsync().ConfigureAwait(false); - Console.WriteLine("trace: " + trace); + using var process = Process.GetCurrentProcess(); + return new ServerStats( + GC.GetTotalAllocatedBytes(true), + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + GC.GetTotalPauseDuration(), + process.TotalProcessorTime, + Monitor.LockContentionCount, + Stopwatch.GetTimestamp(), + ThreadPool.ThreadCount, + process.WorkingSet64, + GC.GetGCMemoryInfo().HeapSizeBytes); + } + + public string Delta(ServerStats mark) + { + double seconds = Stopwatch.GetElapsedTime(mark.Timestamp, Timestamp).TotalSeconds; + TimeSpan cpu = Cpu - mark.Cpu; + return FormattableString.Invariant( + $"sec={seconds:F2} allocMB={(Allocated - mark.Allocated) / 1048576.0:F1} gen0={Gen0 - mark.Gen0} ") + + FormattableString.Invariant( + $"gen1={Gen1 - mark.Gen1} gen2={Gen2 - mark.Gen2} pauseMs={(Pause - mark.Pause).TotalMilliseconds:F0} ") + + FormattableString.Invariant( + $"cpuMs={cpu.TotalMilliseconds:F0} cpuCores={cpu.TotalSeconds / seconds:F2} ") + + FormattableString.Invariant( + $"contention={Contention - mark.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); } + } - var sb = new StringBuilder(); - sb.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionsCount} {string.Join(' ', args.Skip(1))}\n"); - sb.Append(" client: ").Append(clientReport).Append('\n'); - sb.Append(" server: ").Append(report[7..]).Append('\n'); - double ops = workload.Operations; - if (ops > 0) + /// + /// Latency samples (Stopwatch ticks) recorded by many threads. ConcurrentBag keeps a + /// list per thread, so recording does not contend between the load threads. + /// + internal sealed class LatencyRecorder + { + public void Record(long ticks) { - ServerReport sr = ServerReport.Parse(report[7..]); - sb.Append(CultureInfo.InvariantCulture, $" per-op: allocB={sr.AllocMb * 1048576 / ops:F0} cpuUs={sr.CpuMs * 1000 / ops:F1} ({workload.OperationName})\n"); + m_samples.Add(ticks); } - Console.Write(sb.ToString()); - if (s_args.TryGetValue("out", out string? outFile)) + + public void Clear() { - File.AppendAllText(outFile, sb.ToString()); + m_samples.Clear(); } - await Parallel.ForEachAsync(sessions, async (s, ct) => + public string Percentiles() { - try + var all = new List(m_samples); + if (all.Count == 0) { - await s.CloseAsync(ct).ConfigureAwait(false); + return "n=0"; } - catch + all.Sort(); + double Ms(double quantile) { + return all[Math.Min(all.Count - 1, (int)(quantile * all.Count))] * 1000.0 / Stopwatch.Frequency; } - s.Dispose(); - }).ConfigureAwait(false); - server.StandardInput.WriteLine("quit"); - if (!server.WaitForExit(10000)) - { - server.Kill(); + return FormattableString.Invariant( + $"n={all.Count} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms ") + + FormattableString.Invariant($"p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); } - return 0; + + private readonly ConcurrentBag m_samples = []; } - private static string Take(BlockingCollection lines, string prefix) + /// + /// A load pattern driven against the server. + /// + internal abstract class Workload { - while (true) - { - string l = lines.Take(); - if (l.StartsWith(prefix, StringComparison.Ordinal)) - { - return l; - } - } + public abstract double Operations { get; } + + public abstract string OperationName { get; } + + public abstract Task SetupAsync(); + + public abstract Task RunAsync(CancellationToken ct); + + public abstract void ResetStats(); + + public abstract string Report(double seconds); } - private sealed record ServerReport(double AllocMb, double CpuMs) + /// + /// Closed-loop Read, Write or Browse requests from every session. + /// + internal sealed class RequestWorkload : Workload { - public static ServerReport Parse(string s) + public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) { - var d = s.Split(' ').Select(p => p.Split('=')).Where(p => p.Length == 2).ToDictionary(p => p[0], p => p[1]); - return new ServerReport(double.Parse(d["allocMB"], CultureInfo.InvariantCulture), double.Parse(d["cpuMs"], CultureInfo.InvariantCulture)); + m_kind = kind; + m_sessions = sessions; + m_nodes = nodes; + m_inflight = inflight; } - } - internal static List GetNodes(ISession session, int count, Type? onlyType = null) - { - var result = new List(); - foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + public override double Operations => Interlocked.Read(ref m_requests); + + public override string OperationName => "request"; + + public override Task SetupAsync() { - if (onlyType != null && type != onlyType) + List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); + m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); + m_write = nodes.Select(n => new WriteValue { - continue; - } - foreach (ExpandedNodeId e in ids) + NodeId = n, + AttributeId = Attributes.Value, + Value = new DataValue(new Variant(42)) + }).ToArray(); + m_browse = new[] { - if (result.Count == count) + new BrowseDescription { - return result; + NodeId = ObjectIds.Server, + BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, + IncludeSubtypes = true, + ResultMask = (uint)BrowseResultMask.All } - result.Add(ExpandedNodeId.ToNodeId(e, session.NamespaceUris)); - } + }; + Console.WriteLine(FormattableString.Invariant( + $"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session")); + return Task.CompletedTask; } - return result; - } -} -internal abstract class Workload -{ - public abstract Task SetupAsync(); - public abstract Task RunAsync(CancellationToken ct); - public abstract void ResetStats(); - public abstract string Report(double seconds); - public abstract double Operations { get; } - public abstract string OperationName { get; } - - protected static string Percentiles(List ticksList) - { - if (ticksList.Count == 0) + public override async Task RunAsync(CancellationToken ct) { - return "n=0"; + var tasks = new List(); + foreach (ISession session in m_sessions) + { + for (int i = 0; i < m_inflight; i++) + { + tasks.Add(Task.Run(() => LoopAsync(session, ct), CancellationToken.None)); + } + } + await Task.WhenAll(tasks).ConfigureAwait(false); } - long[] a = [.. ticksList]; - Array.Sort(a); - double Ms(double q) => a[Math.Min(a.Length - 1, (int)(q * a.Length))] * 1000.0 / Stopwatch.Frequency; - return string.Create(CultureInfo.InvariantCulture, - $"n={a.Length} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); - } -} -internal sealed class RequestWorkload : Workload -{ - private readonly string m_kind; - private readonly ISession[] m_sessions; - private readonly int m_nodes; - private readonly int m_inflight; - private readonly ConcurrentBag> m_lat = new(); - private readonly ThreadLocal> m_local; - private volatile int m_epoch; - private long m_requests; - private long m_errors; - private ArrayOf m_read; - private ArrayOf m_write; - private ArrayOf m_browse; - - public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) - { - m_kind = kind; - m_sessions = sessions; - m_nodes = nodes; - m_inflight = inflight; - m_local = new ThreadLocal>(() => + public override void ResetStats() { - var l = new List(1 << 16); - m_lat.Add(l); - return l; - }, false); - } - - public override double Operations => Interlocked.Read(ref m_requests); - public override string OperationName => "request"; - - public override Task SetupAsync() - { - List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); - m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); - m_write = nodes.Select(n => new WriteValue { NodeId = n, AttributeId = Attributes.Value, Value = new DataValue(new Variant(42)) }).ToArray(); - m_browse = new[] { new BrowseDescription { NodeId = Opc.Ua.ObjectIds.Server, BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All } }; - Console.WriteLine($"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session"); - return Task.CompletedTask; - } + Interlocked.Increment(ref m_epoch); + m_latency.Clear(); + Interlocked.Exchange(ref m_requests, 0); + Interlocked.Exchange(ref m_errors, 0); + } - public override async Task RunAsync(CancellationToken ct) - { - var tasks = new List(); - foreach (ISession s in m_sessions) + public override string Report(double seconds) { - for (int i = 0; i < m_inflight; i++) - { - tasks.Add(Task.Run(() => LoopAsync(s, ct), CancellationToken.None)); - } + long requests = Interlocked.Read(ref m_requests); + int valuesPerRequest = m_kind == "browse" ? 1 : m_nodes; + return FormattableString.Invariant( + $"req/s={requests / seconds:F0} values/s={requests * valuesPerRequest / seconds:F0} ") + + FormattableString.Invariant($"errors={Interlocked.Read(ref m_errors)} latency {m_latency.Percentiles()}"); } - await Task.WhenAll(tasks).ConfigureAwait(false); - } - private async Task LoopAsync(ISession s, CancellationToken ct) - { - while (!ct.IsCancellationRequested) + private async Task LoopAsync(ISession session, CancellationToken ct) { - long start = Stopwatch.GetTimestamp(); - int epoch = m_epoch; - try + while (!ct.IsCancellationRequested) { - switch (m_kind) + long start = Stopwatch.GetTimestamp(); + int epoch = Volatile.Read(ref m_epoch); + try { - case "write": - await s.WriteAsync(null, m_write, ct).ConfigureAwait(false); - break; - case "browse": - await s.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); - break; - default: - await s.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); - break; + switch (m_kind) + { + case "write": + await session.WriteAsync(null, m_write, ct).ConfigureAwait(false); + break; + case "browse": + await session.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); + break; + default: + await session.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); + break; + } } - } - catch (OperationCanceledException) - { - return; - } - catch (Exception ex) - { - if (Interlocked.Increment(ref m_errors) < 5) + catch (OperationCanceledException) { - Console.WriteLine("error: " + ex.Message); + return; } - continue; - } - long el = Stopwatch.GetTimestamp() - start; - if (epoch == m_epoch) - { - List l = m_local.Value!; - lock (l) + catch (ServiceResultException ex) { - l.Add(el); + if (Interlocked.Increment(ref m_errors) < 5) + { + Console.WriteLine("error: " + ex.Message); + } + continue; + } + if (epoch == Volatile.Read(ref m_epoch)) + { + m_latency.Record(Stopwatch.GetTimestamp() - start); + Interlocked.Increment(ref m_requests); } - Interlocked.Increment(ref m_requests); } } + + private readonly string m_kind; + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_inflight; + private readonly LatencyRecorder m_latency = new(); + private int m_epoch; + private long m_requests; + private long m_errors; + private ArrayOf m_read; + private ArrayOf m_write; + private ArrayOf m_browse; } - public override void ResetStats() + /// + /// Data-change subscriptions on every session and one writer changing the monitored values. + /// The written value is a counter, so each notification maps back to the time of its write. + /// + internal sealed class SubscribeWorkload : Workload { - m_epoch++; - foreach (List l in m_lat) + public SubscribeWorkload(ISession[] sessions, int nodes, int subscriptions, int publishingInterval, int writeInterval) { - lock (l) - { - l.Clear(); - } + m_sessions = sessions; + m_nodes = nodes; + m_subscriptions = subscriptions; + m_publishingInterval = publishingInterval; + m_writeInterval = writeInterval; } - Interlocked.Exchange(ref m_requests, 0); - Interlocked.Exchange(ref m_errors, 0); - } - public override string Report(double seconds) - { - var all = new List(); - foreach (List l in m_lat) + public override double Operations => Interlocked.Read(ref m_notifications); + + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); + var setupTime = Stopwatch.StartNew(); + await Parallel.ForEachAsync( + m_sessions, + new ParallelOptions { MaxDegreeOfParallelism = 16 }, + async (session, ct) => + { + for (int i = 0; i < m_subscriptions; i++) + { + await CreateSubscriptionAsync(session, ct).ConfigureAwait(false); + } + }).ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"sub: {m_sessions.Length} sessions x {m_subscriptions} subs x {m_nodeIds.Count} items, ") + + FormattableString.Invariant( + $"pub {m_publishingInterval} ms, write every {m_writeInterval} ms; setup {setupTime.ElapsedMilliseconds} ms")); + } + + public override async Task RunAsync(CancellationToken ct) { - lock (l) + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); + while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) { - all.AddRange(l); + int counter = Interlocked.Increment(ref m_counter); + var values = new WriteValue[m_nodeIds.Count]; + for (int i = 0; i < values.Length; i++) + { + values[i] = new WriteValue + { + NodeId = m_nodeIds[i], + AttributeId = Attributes.Value, + Value = new DataValue(new Variant(counter)) + }; + } + Volatile.Write(ref m_writeTicks[counter & 0xFFFF], Stopwatch.GetTimestamp()); + await m_sessions[0].WriteAsync(null, values, ct).ConfigureAwait(false); + Interlocked.Increment(ref m_writes); } } - long req = Interlocked.Read(ref m_requests); - return string.Create(CultureInfo.InvariantCulture, - $"req/s={req / seconds:F0} values/s={req * (m_kind == "browse" ? 1 : m_nodes) / seconds:F0} errors={m_errors} latency {Percentiles(all)}"); - } -} -internal sealed class SubscribeWorkload : Workload -{ - private readonly ISession[] m_sessions; - private readonly int m_nodes; - private readonly int m_subs; - private readonly int m_pub; - private readonly int m_writeInterval; - private readonly long[] m_writeTicks = new long[1 << 16]; - private readonly ConcurrentBag> m_lat = new(); - private readonly ThreadLocal> m_local; - private long m_notifications; - private long m_writes; - private int m_counter; - private ISession? m_writer; - private ArrayOf m_nodeIds; - - public SubscribeWorkload(ISession[] sessions, int nodes, int subs, int pub, int writeInterval) - { - m_sessions = sessions; - m_nodes = nodes; - m_subs = subs; - m_pub = pub; - m_writeInterval = writeInterval; - m_local = new ThreadLocal>(() => + public override void ResetStats() { - var l = new List(1 << 16); - m_lat.Add(l); - return l; - }, false); - } + m_latency.Clear(); + Interlocked.Exchange(ref m_notifications, 0); + Interlocked.Exchange(ref m_writes, 0); + } - public override double Operations => Interlocked.Read(ref m_notifications); - public override string OperationName => "notification"; + public override string Report(double seconds) + { + long notifications = Interlocked.Read(ref m_notifications); + long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subscriptions * m_nodeIds.Count; + double ratio = expected == 0 ? 0 : (double)notifications / expected; + return FormattableString.Invariant( + $"notif/s={notifications / seconds:F0} received/expected={ratio:P1} ") + + FormattableString.Invariant($"write->notify latency {m_latency.Percentiles()}"); + } - public override async Task SetupAsync() - { - m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); - m_writer = m_sessions[0]; - var sw = Stopwatch.StartNew(); - await Parallel.ForEachAsync(m_sessions, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (session, ct) => + private async Task CreateSubscriptionAsync(ISession session, CancellationToken ct) { - for (int j = 0; j < m_subs; j++) +#pragma warning disable CA2000 // ownership transfers to the session in AddSubscription; disposed below on failure before that + var subscription = new Subscription(session.DefaultSubscription) { - var subscription = new Subscription(session.DefaultSubscription) - { - PublishingInterval = m_pub, - KeepAliveCount = 10, - LifetimeCount = 100 - }; - foreach (NodeId n in m_nodeIds) + PublishingInterval = m_publishingInterval, + KeepAliveCount = 10, + LifetimeCount = 100, + FastDataChangeCallback = OnDataChange + }; +#pragma warning restore CA2000 + try + { + foreach (NodeId nodeId in m_nodeIds) { subscription.AddItem(new MonitoredItem(subscription.DefaultItem) { - StartNodeId = n, + StartNodeId = nodeId, AttributeId = Attributes.Value, MonitoringMode = MonitoringMode.Reporting, SamplingInterval = 0, QueueSize = 1 }); } - subscription.FastDataChangeCallback = (sub, notification, _) => - { - long now = Stopwatch.GetTimestamp(); - List l = m_local.Value!; - int count = 0; - lock (l) - { - foreach (MonitoredItemNotification item in notification.MonitoredItems) - { - if (item.Value.WrappedValue.TryGetValue(out int v) && v > 0) - { - long w = Volatile.Read(ref m_writeTicks[v & 0xFFFF]); - if (w != 0) - { - l.Add(now - w); - } - } - count++; - } - } - Interlocked.Add(ref m_notifications, count); - }; + // the session owns and disposes the subscription from here on session.AddSubscription(subscription); - await subscription.CreateAsync(ct).ConfigureAwait(false); } - }).ConfigureAwait(false); - Console.WriteLine($"sub: {m_sessions.Length} sessions x {m_subs} subs x {m_nodeIds.Count} items = {m_sessions.Length * m_subs * m_nodeIds.Count} items, pub {m_pub} ms, write every {m_writeInterval} ms; setup {sw.ElapsedMilliseconds} ms"); - } - - public override async Task RunAsync(CancellationToken ct) - { - using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); - while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) - { - int v = Interlocked.Increment(ref m_counter); - var values = new WriteValue[m_nodeIds.Count]; - for (int i = 0; i < values.Length; i++) + catch { - values[i] = new WriteValue { NodeId = m_nodeIds[i], AttributeId = Attributes.Value, Value = new DataValue(new Variant(v)) }; + subscription.Dispose(); + throw; } - Volatile.Write(ref m_writeTicks[v & 0xFFFF], Stopwatch.GetTimestamp()); - await m_writer!.WriteAsync(null, values, ct).ConfigureAwait(false); - Interlocked.Increment(ref m_writes); + await subscription.CreateAsync(ct).ConfigureAwait(false); } - } - public override void ResetStats() - { - foreach (List l in m_lat) + private void OnDataChange(Subscription subscription, DataChangeNotification notification, ArrayOf stringTable) { - lock (l) + long now = Stopwatch.GetTimestamp(); + int count = 0; + foreach (MonitoredItemNotification item in notification.MonitoredItems) { - l.Clear(); + if (item.Value.WrappedValue.TryGetValue(out int counter) && counter > 0) + { + long written = Volatile.Read(ref m_writeTicks[counter & 0xFFFF]); + if (written != 0) + { + m_latency.Record(now - written); + } + } + count++; } + Interlocked.Add(ref m_notifications, count); } - Interlocked.Exchange(ref m_notifications, 0); - Interlocked.Exchange(ref m_writes, 0); - } - public override string Report(double seconds) - { - var all = new List(); - foreach (List l in m_lat) - { - lock (l) - { - all.AddRange(l); - } - } - long n = Interlocked.Read(ref m_notifications); - long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subs * m_nodeIds.Count; - return string.Create(CultureInfo.InvariantCulture, - $"notif/s={n / seconds:F0} received/expected={(expected == 0 ? 0 : (double)n / expected):P1} write->notify latency {Percentiles(all)}"); + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_subscriptions; + private readonly int m_publishingInterval; + private readonly int m_writeInterval; + private readonly long[] m_writeTicks = new long[1 << 16]; + private readonly LatencyRecorder m_latency = new(); + private long m_notifications; + private long m_writes; + private int m_counter; + private ArrayOf m_nodeIds; } } diff --git a/perf/ServerLoadHarness/ServerLoadHarness.csproj b/perf/ServerLoadHarness/ServerLoadHarness.csproj index 1d238e7b18..6fae6cf2bb 100644 --- a/perf/ServerLoadHarness/ServerLoadHarness.csproj +++ b/perf/ServerLoadHarness/ServerLoadHarness.csproj @@ -1,17 +1,14 @@ + Exe net10.0 - enable - enable false + false true - true - $(NoWarn);CS1591;CA1303;CA2007;CA1031;CA2000;CA1849;CA5394;RCS1090;CA1515;CA1305;CA1307;CA1310;IDE0005;CA1062;CA1822;CA1859;CA1861;CS0618;RCS1163;CA1002;CA1812;CA1852;CA1050;CA2201;CA1001 - false - false - none diff --git a/perf/TraceAnalyzer/Program.cs b/perf/TraceAnalyzer/Program.cs index a8002ec6eb..175d9de63d 100644 --- a/perf/TraceAnalyzer/Program.cs +++ b/perf/TraceAnalyzer/Program.cs @@ -1,174 +1,276 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + // Trace analyzer for perf/ServerLoadHarness traces. Not part of the product; see perf/README.md. -// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped), exclusive/inclusive, callers of focus -// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, by first Opc.Ua frame, and stacks -// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack +// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped) +// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, first Opc.Ua frame, stacks +// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Etlx; using Microsoft.Diagnostics.Tracing.Parsers.Clr; -string mode = args[0]; -string file = args[1]; -int top = args.Length > 2 ? int.Parse(args[2]) : 40; -string focus = args.Length > 3 ? args[3] : null; -string[] s_blocked = { "LowLevelLifoSemaphore.Wait", "WaitForSignal", "WorkerThreadStart", "IOCompletionPoller.Poll", "WaitOneNoCheck", "Thread.Sleep", "PollGCWorker|70_0", "WaitMultipleIgnoringSyncContext", "Monitor.Wait", "!?" }; - -string etlx = TraceLog.CreateFromEventPipeDataFile(file, null, new TraceLogOptions { ContinueOnError = true }); -using var log = new TraceLog(etlx); - -static string Name(TraceCallStack s) +namespace Opc.Ua.Perf.TraceAnalyzer { - var m = s.CodeAddress.FullMethodName; - if (string.IsNullOrEmpty(m)) + /// + /// Aggregates EventPipe traces recorded by the server load harness. + /// + public static class Program { - m = s.CodeAddress.ModuleName + "!?"; - } - return m; -} + /// + /// Leaf frames of threads that wait rather than run. The sampled thread-time + /// profile samples blocked threads too, so these are dropped to approximate CPU. + /// + private static readonly string[] s_blockedLeafFrames = + [ + "LowLevelLifoSemaphore.Wait", + "WaitForSignal", + "WorkerThreadStart", + "IOCompletionPoller.Poll", + "WaitOneNoCheck", + "Thread.Sleep", + "PollGCWorker|70_0", + "WaitMultipleIgnoringSyncContext", + "Monitor.Wait", + "!?" + ]; -static string Short(string m) -{ - int p = m.IndexOf('('); - return p > 0 ? m[..p] : m; -} + /// + /// Entry point. + /// + public static int Main(string[] args) + { + if (args.Length < 2) + { + Console.WriteLine("usage: TraceAnalyzer cpu|alloc|contention [top] [focus]"); + return 1; + } -static bool IsOurs(string m) => m.StartsWith("Opc.Ua", StringComparison.Ordinal) || m.StartsWith("Quickstarts", StringComparison.Ordinal); + string mode = args[0]; + int top = args.Length > 2 ? int.Parse(args[2], CultureInfo.InvariantCulture) : 40; + string? focus = args.Length > 3 ? args[3] : null; -if (mode == "contention") -{ - var byStack = new Dictionary(); - int total = 0; - foreach (TraceEvent e in log.Events) - { - if (e.EventName == null || !e.EventName.StartsWith("Contention/Start")) continue; - total++; - var chain = new List(); - for (TraceCallStack f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) { string n = Short(Name(f)); if (IsOurs(n) || chain.Count == 0) chain.Add(n); } - string k = string.Join("\n ", chain); - byStack[k] = byStack.GetValueOrDefault(k) + 1; - } - Console.WriteLine($"contention events: {total}"); - foreach (var kv in byStack.OrderByDescending(k => k.Value).Take(top)) Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); - return; -} -if (mode == "cpu") -{ - var excl = new Dictionary(); - var incl = new Dictionary(); - var firstOurs = new Dictionary(); - var callers = new Dictionary(); - int total = 0, all = 0; - foreach (TraceEvent e in log.Events) - { - if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") - { - continue; - } - all++; - object type = e.PayloadByName("Type"); - if (type != null && Convert.ToInt32(type) != 2) - { - continue; // not running managed code - } - TraceCallStack s = e.CallStack(); - if (s == null) - { - continue; + string etlx = TraceLog.CreateFromEventPipeDataFile( + args[1], null, new TraceLogOptions { ContinueOnError = true }); + using var log = new TraceLog(etlx); + + switch (mode) + { + case "contention": + Contention(log, top); + break; + case "cpu": + Cpu(log, top, focus); + break; + default: + Allocations(log, top, focus); + break; + } + return 0; } - string leaf = Short(Name(s)); - if (s_blocked.Any(b => leaf.EndsWith(b))) continue; - total++; - excl[leaf] = excl.GetValueOrDefault(leaf) + 1; - var seen = new HashSet(); - bool gotOurs = false; - bool focusHit = false; - for (TraceCallStack f = s; f != null; f = f.Caller) + + private static void Contention(TraceLog log, int top) { - string n = Short(Name(f)); - if (seen.Add(n)) + var byStack = new Dictionary(StringComparer.Ordinal); + double total = 0; + foreach (TraceEvent e in log.Events) { - incl[n] = incl.GetValueOrDefault(n) + 1; + if (e.EventName?.StartsWith("Contention/Start", StringComparison.Ordinal) != true) + { + continue; + } + total++; + var chain = new List(); + for (TraceCallStack? f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) + { + string n = Short(Name(f)); + if (IsOurs(n) || chain.Count == 0) + { + chain.Add(n); + } + } + Add(byStack, string.Join("\n ", chain), 1); } - if (!gotOurs && IsOurs(n)) + Console.WriteLine(FormattableString.Invariant($"contention events: {total}")); + Dump("stacks", byStack, total, top); + } + + private static void Cpu(TraceLog log, int top, string? focus) + { + var exclusive = new Dictionary(StringComparer.Ordinal); + var inclusive = new Dictionary(StringComparer.Ordinal); + var firstOurs = new Dictionary(StringComparer.Ordinal); + var callers = new Dictionary(StringComparer.Ordinal); + double total = 0; + int all = 0; + foreach (TraceEvent e in log.Events) { - gotOurs = true; - firstOurs[n] = firstOurs.GetValueOrDefault(n) + 1; + if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") + { + continue; + } + all++; + object? type = e.PayloadByName("Type"); + if (type != null && Convert.ToInt32(type, CultureInfo.InvariantCulture) != 2) + { + // not running managed code + continue; + } + TraceCallStack? stack = e.CallStack(); + if (stack == null) + { + continue; + } + string leaf = Short(Name(stack)); + if (s_blockedLeafFrames.Any(b => leaf.EndsWith(b, StringComparison.Ordinal))) + { + continue; + } + total++; + Add(exclusive, leaf, 1); + var seen = new HashSet(StringComparer.Ordinal); + bool gotOurs = false; + bool focusHit = false; + for (TraceCallStack? f = stack; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (seen.Add(n)) + { + Add(inclusive, n, 1); + } + if (!gotOurs && IsOurs(n)) + { + gotOurs = true; + Add(firstOurs, n, 1); + } + if (focus != null && !focusHit && n.Contains(focus, StringComparison.Ordinal)) + { + focusHit = true; + Add(callers, f.Caller != null ? Short(Name(f.Caller)) : "", 1); + } + } } - if (focus != null && !focusHit && n.Contains(focus)) + Console.WriteLine(FormattableString.Invariant($"samples: on-cpu={total} all={all}")); + Dump("exclusive", exclusive, total, top); + Dump("first Opc.Ua frame", firstOurs, total, top); + Dump( + "inclusive (Opc.Ua only)", + inclusive.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal), + total, + top); + if (focus != null) { - focusHit = true; - string c = f.Caller != null ? Short(Name(f.Caller)) : ""; - callers[c] = callers.GetValueOrDefault(c) + 1; + Dump("callers of " + focus, callers, total, top); } } - } - Console.WriteLine($"samples: managed-running={total} all={all}"); - void Dump(string title, Dictionary d, int n) - { - Console.WriteLine($"--- {title}"); - foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) - { - Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); - } - } - Dump("exclusive", excl, top); - Dump("first Opc.Ua frame (exclusive-ish)", firstOurs, top); - Dump("inclusive (Opc.Ua only)", incl.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value), top); - if (focus != null) - { - Dump("callers of " + focus, callers, top); - } -} -else -{ - var byType = new Dictionary(); - var byFrame = new Dictionary(); - var byStack = new Dictionary(); - double total = 0; - foreach (TraceEvent e in log.Events) - { - if (e is not GCAllocationTickTraceData a) - { - continue; - } - double amount = a.AllocationAmount64; - total += amount; - string t = a.TypeName ?? "?"; - byType[t] = byType.GetValueOrDefault(t) + amount; - TraceCallStack s = e.CallStack(); - string frame = "?"; - var chain = new List(); - for (TraceCallStack f = s; f != null; f = f.Caller) + + private static void Allocations(TraceLog log, int top, string? focus) { - string n = Short(Name(f)); - if (IsOurs(n)) + var byType = new Dictionary(StringComparer.Ordinal); + var byFrame = new Dictionary(StringComparer.Ordinal); + var byStack = new Dictionary(StringComparer.Ordinal); + double total = 0; + foreach (TraceEvent e in log.Events) { - if (frame == "?") + if (e is not GCAllocationTickTraceData tick) + { + continue; + } + double amount = tick.AllocationAmount64; + total += amount; + string type = tick.TypeName ?? "?"; + Add(byType, type, amount); + string frame = "?"; + var chain = new List(); + for (TraceCallStack? f = e.CallStack(); f != null; f = f.Caller) { - frame = n; + string n = Short(Name(f)); + if (!IsOurs(n)) + { + continue; + } + if (frame == "?") + { + frame = n; + } + if (chain.Count < 6 || (focus != null && chain.Count < 12)) + { + chain.Add(n); + } } - if (chain.Count < 6) + Add(byFrame, type + " <- " + frame, amount); + if (focus == null || type.Contains(focus, StringComparison.Ordinal)) { - chain.Add(n); + Add(byStack, type + "\n " + string.Join("\n ", chain), amount); } } + Console.WriteLine(FormattableString.Invariant($"sampled alloc total: {total / 1048576:F1} MB")); + Dump("by type", byType, total, top); + Dump("by type <- first Opc.Ua frame", byFrame, total, top); + Dump("stacks", byStack, total, top / 2); } - byFrame[t + " <- " + frame] = byFrame.GetValueOrDefault(t + " <- " + frame) + amount; - if (focus == null || t.Contains(focus)) + + private static void Add(Dictionary totals, string key, double amount) { - string k = t + "\n " + string.Join("\n ", chain); - byStack[k] = byStack.GetValueOrDefault(k) + amount; + totals[key] = totals.GetValueOrDefault(key) + amount; } - } - Console.WriteLine($"sampled alloc total: {total / 1048576:F1} MB"); - void Dump(string title, Dictionary d, int n) - { - Console.WriteLine($"--- {title}"); - foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + + private static void Dump(string title, Dictionary totals, double total, int count) + { + Console.WriteLine("--- " + title); + foreach (KeyValuePair kv in totals.OrderByDescending(k => k.Value).Take(count)) + { + Console.WriteLine(FormattableString.Invariant($"{100.0 * kv.Value / total,6:F2}% {kv.Key}")); + } + } + + private static string Name(TraceCallStack frame) + { + string name = frame.CodeAddress.FullMethodName; + return string.IsNullOrEmpty(name) ? frame.CodeAddress.ModuleName + "!?" : name; + } + + private static string Short(string method) + { + int p = method.IndexOf('(', StringComparison.Ordinal); + return p > 0 ? method[..p] : method; + } + + private static bool IsOurs(string method) { - Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + return method.StartsWith("Opc.Ua", StringComparison.Ordinal) || + method.StartsWith("Quickstarts", StringComparison.Ordinal); } } - Dump("by type", byType, top); - Dump("by type <- first Opc.Ua frame", byFrame, top); - Dump("stacks", byStack, top / 2); } diff --git a/perf/TraceAnalyzer/TraceAnalyzer.csproj b/perf/TraceAnalyzer/TraceAnalyzer.csproj index e2529b9c27..b43a522d76 100644 --- a/perf/TraceAnalyzer/TraceAnalyzer.csproj +++ b/perf/TraceAnalyzer/TraceAnalyzer.csproj @@ -3,13 +3,7 @@ Exe net10.0 - disable - enable false - false - false - none - $(NoWarn);CS1591;CA1305;CA1307;CA1310;CA1849;RCS1007;IDE0011 From dc638ab9fae778e6fb3c10c4eb6632dfb91594f1 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:21:43 +0200 Subject: [PATCH 07/16] Fix certificate checks after the ByteString migration; cover queue and pool edge cases - StandardServer compared the session certificate with the channel certificate through Utils.IsEqual, which bound to the object overload once the channel certificate became a ByteString and always failed (BadSecurityChecksFailed on secure CreateSession and TransferSubscriptions). Compare the spans. - The isolation provider's ChannelSnapshot copies the certificate again: the context wraps a buffer the transport owns, and revalidation must detect evidence that changed after the owner was issued. - PeerMatches has the array fallback only on .NET Framework; on .NET the 16-byte stack buffer always holds the address. - Tests for a stopped FairRequestQueue (admission and readers) and for the ObjectPool capacity argument. Co-Authored-By: Claude Opus 5.5 --- .../DefaultServerResourceIsolationProvider.cs | 15 +++++----- src/Opc.Ua.Server/Server/StandardServer.cs | 5 ++-- .../Stack/Server/FairRequestQueueTests.cs | 30 +++++++++++++++++++ .../Types/Utils/ObjectPoolTests.cs | 9 ++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index 85fb0babca..60d2b7deba 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -844,9 +844,10 @@ public ChannelSnapshot(SecureChannelContext context) m_channelId = context.SecureChannelId; SecurityPolicyUri = context.EndpointDescription?.SecurityPolicyUri; SecurityMode = context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid; - // ByteString is a read-only view, so the evidence cannot change after capture. + // Copied: the context wraps a buffer the transport owns, and revalidation + // must detect evidence that changed after the owner was issued. m_certificate = SecurityPolicyUri != SecurityPolicies.None - ? context.ClientChannelCertificate : default; + ? ByteString.From(context.ClientChannelCertificate.Span) : default; m_peer = context.PeerAddress?.GetAddressBytes(); m_upstreamIdentity = context.UpstreamIdentity; } @@ -886,13 +887,13 @@ private bool PeerMatches(IPAddress? peer) return m_peer == null || m_peer.Length == 0; } #if NET + // 16 bytes hold an IPv6 address, so the write always succeeds. Span bytes = stackalloc byte[16]; - if (peer.TryWriteBytes(bytes, out int written)) - { - return m_peer.AsSpan().SequenceEqual(bytes[..written]); - } -#endif + return peer.TryWriteBytes(bytes, out int written) && + m_peer.AsSpan().SequenceEqual(bytes[..written]); +#else return m_peer.AsSpan().SequenceEqual(peer.GetAddressBytes()); +#endif } private readonly string m_channelId; diff --git a/src/Opc.Ua.Server/Server/StandardServer.cs b/src/Opc.Ua.Server/Server/StandardServer.cs index e9aedf6b6e..8b67566052 100644 --- a/src/Opc.Ua.Server/Server/StandardServer.cs +++ b/src/Opc.Ua.Server/Server/StandardServer.cs @@ -751,9 +751,8 @@ public override async ValueTask CreateSessionAsync( { string? profile = context.ChannelContext.EndpointDescription!.TransportProfileUri; if (profile is Profiles.UaTcpTransport or Profiles.UaWssTransport && - !Utils.IsEqual( - parsedClientCertificate!.RawData, - context.ChannelContext.ClientChannelCertificate)) + !context.ChannelContext.ClientChannelCertificate.Span.SequenceEqual( + parsedClientCertificate!.RawData)) { throw ServiceResultException.Create( StatusCodes.BadSecurityChecksFailed, diff --git a/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs index 3946f27942..da4e1f6cf4 100644 --- a/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs +++ b/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs @@ -70,6 +70,36 @@ public void WeightedRoundsPreserveOwnerFifo() Assert.That(provider.Used(ResourceIsolationStage.RequestQueueBytes), Is.Zero); } + /// + /// A stopped queue rejects admission before classifying or charging the request. + /// + [Test] + public void StoppedQueueRejectsAdmissionWithoutChargingTheProvider() + { + var provider = new TestProvider(); + NodeId token = provider.AddOwner("A"); + FairRequestQueue queue = CreateQueue(provider); + queue.Dispose(); + + Assert.That(queue.TryEnqueue(new TestRequest(token), default, out StatusCode status), Is.False); + Assert.That(status, Is.EqualTo(StatusCodes.BadServerHalted)); + Assert.That(provider.TotalUsed, Is.Zero); + } + + /// + /// A reader that arrives after the queue stopped is cancelled instead of waiting. + /// + [Test] + public void StoppedQueueCancelsANewReader() + { + var provider = new TestProvider(); + FairRequestQueue queue = CreateQueue(provider); + queue.Dispose(); + + Assert.ThrowsAsync( + async () => await queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false)); + } + /// /// FIFO ordering ignores owner weights and preserves interleaved global admission order. /// diff --git a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs index 66584127de..c50e5819df 100644 --- a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs +++ b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs @@ -45,6 +45,15 @@ namespace Opc.Ua.Core.Tests.Types.UtilsTests [Parallelizable] public class ObjectPoolTests { + /// + /// A pool must be able to retain at least one object. + /// + [Test] + public void NonPositiveCapacityIsRejected() + { + Assert.Throws(() => _ = new ObjectPool(() => new object(), 0)); + } + /// /// A returned object is handed out again before a new one is created. /// From 30b769d13080ce4d79e67ec0ef2d63ae0b592c62 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 08/16] Server: sample at a fixed rate instead of a fixed delay after each sample The sampling loop waited one interval after every sample, so the timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on Windows) and the sampling time were added to every cycle: a 10 ms group took 63 % (Windows) to 90 % (Linux) of its samples, and the o6-automation opcua-benchmarks subscription test (100 Hz, every item >= 99 % of samples) failed with a single monitored item. Samples now have absolute deadlines one interval apart; a late wake-up shortens the next wait, and a stall longer than four intervals (at least 50 ms) restarts the schedule instead of bursting. The benchmark workload now passes up to 4096 items (about 410k samples/s). NodeState.ReadAttributes fills the result array in place instead of copying a list, removing an allocation per sampled permission check. Co-Authored-By: Claude Opus 5.5 --- .../MonitoredItem/SamplingGroup.cs | 46 +++++++++---- src/Opc.Ua.Types/State/NodeState.cs | 7 +- .../SamplingGroupLoopRegressionTests.cs | 67 +++++++++++++++++++ 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 298259a988..287e5a7ef4 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -570,7 +570,19 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance m_logger.ServerNameThreadStarted(Thread.CurrentThread.Name); int sleepCycle = Convert.ToInt32(samplingInterval, CultureInfo.InvariantCulture); - int timeToWait = sleepCycle; + + // Fixed-rate schedule: each sample has an absolute deadline one interval after + // the previous one. Waiting a fixed interval after each sample instead adds the + // timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on + // Windows) and the sampling time to every cycle, so a 10 ms group sampled about + // 80 to 90 times per second. A late wake-up shortens the next wait, which keeps + // the average rate at the requested interval. + long frequency = m_timeProvider.TimestampFrequency; + long period = Math.Max(1, (long)(samplingInterval * frequency / 1000.0)); + // Missed deadlines are made up by sampling again right away, but not after a + // stall longer than this: the schedule restarts instead of bursting. + long maxLag = Math.Max(4 * period, frequency / 20); + long deadline = m_timeProvider.GetTimestamp() + period; while (m_server.IsRunning && !cancellationToken.IsCancellationRequested) { @@ -579,7 +591,19 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance // wait till next sample without holding a thread. try { - await Task.Delay(timeToWait, cancellationToken).ConfigureAwait(false); + long remaining = deadline - startTimestamp; + if (remaining > 0) + { + await Task.Delay( + TimeSpan.FromTicks(remaining * TimeSpan.TicksPerSecond / frequency), + cancellationToken).ConfigureAwait(false); + } + else + { + // catching up: let other work run between back-to-back samples. + await Task.Yield(); + cancellationToken.ThrowIfCancellationRequested(); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -630,18 +654,14 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance break; } - int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; - timeToWait = sleepCycle; - - if (delay > sleepCycle) + deadline += period; + long now = m_timeProvider.GetTimestamp(); + if (now - deadline > maxLag) { - timeToWait = (2 * sleepCycle) - delay; - - if (timeToWait < 0) - { - m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); - timeToWait = sleepCycle; - } + // the samples take longer than the interval, or the process stalled + int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; + m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); + deadline = now + period; } } diff --git a/src/Opc.Ua.Types/State/NodeState.cs b/src/Opc.Ua.Types/State/NodeState.cs index 0fb8db518f..f8375f415c 100644 --- a/src/Opc.Ua.Types/State/NodeState.cs +++ b/src/Opc.Ua.Types/State/NodeState.cs @@ -4295,7 +4295,8 @@ public virtual ArrayOf ReadAttributes( return []; } - var values = new List(attributeIds.Length); + // filled in place and wrapped without a copy; a list would be copied again. + var values = new Variant[attributeIds.Length]; var scratch = new DataValue(); for (int ii = 0; ii < attributeIds.Length; ii++) @@ -4311,10 +4312,10 @@ public virtual ArrayOf ReadAttributes( default, ref scratch); - values.Add(ServiceResult.IsBad(result) ? default : scratch.WrappedValue); + values[ii] = ServiceResult.IsBad(result) ? default : scratch.WrappedValue; } - return values.ToArrayOf(); + return values; } /// diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index ad603fe015..e6172e1415 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -430,6 +430,73 @@ public async Task CustomItemIsSampledImmediatelyOnlyWithoutQueuedInitialValueAsy } } + /// + /// The loop samples at the requested rate on average. A loop that waits one interval + /// after each sample adds the timer overshoot and the sampling time to every cycle and + /// took 63 % (Windows, 15.6 ms timer) to 90 % (Linux) of the samples of a 10 ms group. + /// + [Test] + [NonParallelizable] + public async Task SamplingKeepsTheRequestedRateAsync() + { + Mock server = DeterministicServerMock.Create( + out MonitoredItemQueueFactory queueFactory); + using (queueFactory) + { + int reads = 0; + var nodeManager = new Mock(); + nodeManager + .Setup(m => m.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Returns, IList, + IList, CancellationToken>( + (_, _, _, values, _, _) => + { + Interlocked.Increment(ref reads); + for (int ii = 0; ii < values.Count; ii++) + { + values[ii] = new DataValue(new Variant(1)); + } + return default; + }); + nodeManager + .Setup(m => m.ValidateRolePermissionsAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(new ValueTask(ServiceResult.Good)); + + using OperationContext context = CreateContext(); + using var group = new SamplingGroup( + server.Object, nodeManager.Object, [new SamplingRateGroup(10, 1, 0)], context, 10); + Mock item = CreateItem(1, 10); + Assert.That(group.StartMonitoring(context, item.Object), Is.True); + group.ApplyChanges(); + + // let the loop start, then count the samples of a fixed window + await Task.Delay(300).ConfigureAwait(false); + int start = Volatile.Read(ref reads); + var window = System.Diagnostics.Stopwatch.StartNew(); + await Task.Delay(2000).ConfigureAwait(false); + int samples = Volatile.Read(ref reads) - start; + double expected = window.Elapsed.TotalMilliseconds / 10; + + group.StopMonitoring(item.Object); + group.ApplyChanges(); + + Assert.That(samples / expected, Is.GreaterThanOrEqualTo(0.95), + $"{samples} samples in {window.Elapsed.TotalMilliseconds:F0} ms at a 10 ms interval"); + Assert.That(samples / expected, Is.LessThanOrEqualTo(1.05), + "the loop must not sample faster than requested"); + } + } + private static int QueuedCount(Mock item) { return item.Invocations.Count(i => i.Method.Name == nameof(IDataChangeMonitoredItem.QueueValue)); From cdac5c5ca4e34461a2a959c053be3a504bdb0175 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 09/16] Server: send responses without blocking pool threads on the channel gate The TCP listeners sent each response through TcpServerChannel.SendResponse, which waits for the channel gate synchronously on the thread-pool worker that finished the request. With many requests in flight per channel, workers parked on the gate while the receive loop that was handed the gate waited for a free pool thread, and the channel stalled until the pool grew: 10 sessions with 100 outstanding Reads each dropped to 0-1.7k req/s (0 with SignAndEncrypt). SendResponseAsync waits for the gate asynchronously; the TCP, Kestrel TCP and WebSocket listeners use it, and the latter two use the cached certificate copies. Same load: 121-131k req/s (89-91k encrypted). Co-Authored-By: Claude Opus 5.5 --- .../Https/HttpsTransportListener.cs | 6 +- .../Tcp/KestrelTcpTransportListener.cs | 8 +- src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs | 37 ++++++ .../Stack/Tcp/TcpTransportListener.cs | 9 +- .../PipelinedRequestTests.cs | 105 ++++++++++++++++++ 5 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs diff --git a/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs b/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs index 0f84429cdc..f08876f2b5 100644 --- a/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs +++ b/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs @@ -2199,8 +2199,8 @@ private async void OnRequestReceivedAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, peerAddress: (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); @@ -2208,7 +2208,7 @@ private async void OnRequestReceivedAsync( .ProcessRequestAsync(context, request) .ConfigureAwait(false); - serverChannel.SendResponse(requestId, response); + await serverChannel.SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (Exception ex) { diff --git a/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs b/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs index 1b1ffcbac4..64d6a9390c 100644 --- a/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs +++ b/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs @@ -923,14 +923,14 @@ private async Task DispatchRequestAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); IServiceResponse response = await m_callback .ProcessRequestAsync(context, request) .ConfigureAwait(false); - ((TcpServerChannel)channel).SendResponse(requestId, response); + await ((TcpServerChannel)channel).SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (Exception ex) { @@ -938,7 +938,7 @@ private async Task DispatchRequestAsync( try { ServiceFault fault = EndpointBase.CreateFault(Logger, request, ex); - ((TcpServerChannel)channel).SendResponse(requestId, fault); + await ((TcpServerChannel)channel).SendResponseAsync(requestId, fault).ConfigureAwait(false); } catch (Exception faultEx) { diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs index 73c634eab1..f192694225 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs @@ -1776,6 +1776,43 @@ public bool SendResponse(uint requestId, IServiceResponse response) } using (Gate.Enter()) + { + return SendResponseCore(requestId, response); + } + } + + /// + /// Sends the response for the specified request, waiting for the channel gate + /// without blocking the calling thread. + /// + /// + /// Used by the request dispatch path, which runs on thread-pool workers. With many + /// requests in flight on one channel, the synchronous + /// parked one pool thread per waiting response on the gate; the receive loop that + /// was handed the gate then waited for a free pool thread to continue, and the + /// channel stalled until the pool injected more threads. + /// + /// true if the response was retained for delivery after reconnect. + /// is null. + /// + public async ValueTask SendResponseAsync(uint requestId, IServiceResponse response) + { + if (response == null) + { + throw new ArgumentNullException(nameof(response)); + } + + using (await Gate.EnterAsync().ConfigureAwait(false)) + { + return SendResponseCore(requestId, response); + } + } + + /// + /// Encodes and queues a response while the caller holds the channel gate. + /// + private bool SendResponseCore(uint requestId, IServiceResponse response) + { { // must queue the response if the channel is in the faulted state. if (State == TcpChannelState.Faulted) diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs index b12a02cd54..8bf1306314 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs @@ -1732,7 +1732,8 @@ private async void OnRequestReceivedAsync( try { - responseRetained = ((TcpServerChannel)channel).SendResponse(requestId, response); + responseRetained = await ((TcpServerChannel)channel) + .SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (ServiceResultException sre) when (sre.StatusCode == StatusCodes.BadSecureChannelClosed) { @@ -1749,7 +1750,8 @@ out uint channelId // if the channel is not the same as the one we started with, send the response over the new channel if (serverChannel != channel) { - responseRetained = serverChannel.SendResponse(requestId, response); + responseRetained = await serverChannel + .SendResponseAsync(requestId, response).ConfigureAwait(false); return; } } @@ -1784,7 +1786,8 @@ out uint channelId ServiceFault fault = EndpointBase.CreateFault(m_logger, request, e); (response as IPooledEncodeable)?.Reuse(); response = fault; - responseRetained = ((TcpServerChannel)channel).SendResponse(requestId, fault); + responseRetained = await ((TcpServerChannel)channel) + .SendResponseAsync(requestId, fault).ConfigureAwait(false); } catch (ServiceResultException faultSre) when (faultSre.StatusCode == StatusCodes.BadSecureChannelClosed) diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs new file mode 100644 index 0000000000..145a362137 --- /dev/null +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -0,0 +1,105 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Client; +using Opc.Ua.Client.TestFramework; + +namespace Opc.Ua.Sessions.Tests +{ + /// + /// Many requests in flight on each session complete without stalling the channels. + /// + [TestFixture] + [Category("SessionServices")] + [NonParallelizable] + public class PipelinedRequestTests : TestFixture + { + /// + protected override void ConfigureServer(ApplicationConfiguration configuration) + { + base.ConfigureServer(configuration); + // admit the whole pipeline: 10 sessions x 90 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 2000; + } + + /// + /// The server used to send each response through a synchronous wait on the channel + /// gate from a thread-pool worker. With many responses ready at once, the workers + /// parked on the gate while the receive loop that was handed the gate waited for a + /// free pool thread, and the channel stalled until the pool grew. 90 Reads stay below + /// the channel's limit of 100 queued writes. + /// + [Test] + public async Task ManyConcurrentReadsOnEachSessionCompleteAsync() + { + ArrayOf nodesToRead = new[] + { + new ReadValueId { NodeId = VariableIds.Server_ServerStatus_CurrentTime, AttributeId = Attributes.Value } + }; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var sessions = new List { Session }; + try + { + for (int ii = 1; ii < 10; ii++) + { + sessions.Add(await ClientFixture.ConnectAsync(ServerUrl, SecurityPolicies.None).ConfigureAwait(false)); + } + var elapsed = Stopwatch.StartNew(); + + for (int round = 0; round < 20; round++) + { + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 90).Select( + _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); + ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); + Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); + } + + // 18000 small Reads take under a second when the channels keep flowing; the + // stalled channel took about 10 s while the pool injected threads. + Assert.That(elapsed.Elapsed, Is.LessThan(TimeSpan.FromSeconds(8))); + } + finally + { + foreach (ISession session in sessions.Skip(1)) + { + await session.CloseAsync(5000, true).ConfigureAwait(false); + session.Dispose(); + } + } + } + } +} From 0db74daf425bb55040ef4a94ce6dfdde32c2e1c2 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 10/16] perf: add the counter workload of the o6-automation subscription benchmark `--scenario counters` runs a server whose UInt64 counters advance in their read callback (sampling groups, configurable sampling rate) and checks per item, on the client and on the server, that at least 99 % of the expected samples arrived. `--queue` sizes MaxQueuedRequestCount for deep pipelines. Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/CounterServer.cs | 162 +++++++++++++++++++ perf/ServerLoadHarness/CounterWorkload.cs | 188 ++++++++++++++++++++++ perf/ServerLoadHarness/Program.cs | 33 +++- 3 files changed, 376 insertions(+), 7 deletions(-) create mode 100644 perf/ServerLoadHarness/CounterServer.cs create mode 100644 perf/ServerLoadHarness/CounterWorkload.cs diff --git a/perf/ServerLoadHarness/CounterServer.cs b/perf/ServerLoadHarness/CounterServer.cs new file mode 100644 index 0000000000..4c72ac546e --- /dev/null +++ b/perf/ServerLoadHarness/CounterServer.cs @@ -0,0 +1,162 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Threading; +using Opc.Ua.Server; + +namespace Opc.Ua.Perf.ServerLoadHarness +{ + /// + /// Counter variables advanced by their read callback, so only sampling drives them. + /// Mirrors the subscription capacity workload of the o6-automation opcua-benchmarks + /// (UInt64 counter per monitored item, sampling groups, read-callback increments). + /// + internal sealed class CounterNodeManager : CustomNodeManager2 + { + public const string Namespace = "urn:opcfoundation.org:perf:counters"; + + public CounterNodeManager(IServerInternal server, ApplicationConfiguration configuration, int count) + : base(server, configuration, useSamplingGroups: true, Namespace) + { + m_values = new ulong[count]; + m_marked = new ulong[count]; + } + + public override void CreateAddressSpace(IDictionary> externalReferences) + { + if (!externalReferences.TryGetValue(ObjectIds.ObjectsFolder, out IList? references)) + { + externalReferences[ObjectIds.ObjectsFolder] = references = new List(); + } + for (int i = 0; i < m_values.Length; i++) + { + int index = i; + var node = new BaseDataVariableState(null) + { + NodeId = new NodeId((uint)i + 1, NamespaceIndex), + BrowseName = new QualifiedName("Counter" + i.ToString(CultureInfo.InvariantCulture), NamespaceIndex), + DisplayName = new LocalizedText("Counter" + i.ToString(CultureInfo.InvariantCulture)), + TypeDefinitionId = VariableTypeIds.BaseDataVariableType, + ReferenceTypeId = ReferenceTypeIds.Organizes, + DataType = DataTypeIds.UInt64, + ValueRank = ValueRanks.Scalar, + MinimumSamplingInterval = 1, + AccessLevel = AccessLevels.CurrentRead, + UserAccessLevel = AccessLevels.CurrentRead, + Value = new Variant((ulong)0), + StatusCode = StatusCodes.Good, + Timestamp = DateTimeUtc.Now + }; + node.OnReadValue = (ISystemContext context, NodeState state, NumericRange range, + QualifiedName encoding, ref Variant value, ref StatusCode status, ref DateTimeUtc timestamp) => + { + value = new Variant(Interlocked.Increment(ref m_values[index])); + status = StatusCodes.Good; + timestamp = DateTimeUtc.Now; + return ServiceResult.Good; + }; + node.AddReference(ReferenceTypeIds.Organizes, true, ObjectIds.ObjectsFolder); + references.Add(new NodeStateReference(ReferenceTypeIds.Organizes, false, node.NodeId)); + AddPredefinedNode(SystemContext, node); + } + } + + /// + /// Starts a measurement window. + /// + public void Mark() + { + for (int i = 0; i < m_values.Length; i++) + { + m_marked[i] = Interlocked.Read(ref m_values[i]); + } + m_markTimestamp = Stopwatch.GetTimestamp(); + } + + /// + /// Reports the sampling progress of the window: completed read callbacks per item + /// against the number the sampling interval calls for. + /// + public string Report(double samplingIntervalMs) + { + double elapsedMs = Stopwatch.GetElapsedTime(m_markTimestamp).TotalMilliseconds; + double expected = elapsedMs / samplingIntervalMs; + double min = double.MaxValue; + double sum = 0; + int belowTarget = 0; + for (int i = 0; i < m_values.Length; i++) + { + double progress = (Interlocked.Read(ref m_values[i]) - m_marked[i]) / expected; + min = Math.Min(min, progress); + sum += progress; + if (progress < 0.99) + { + belowTarget++; + } + } + return FormattableString.Invariant( + $"counters={m_values.Length} samplesExpected={expected:F0} progressMin={min:P2} ") + + FormattableString.Invariant( + $"progressMean={sum / m_values.Length:P2} itemsBelow99={belowTarget}"); + } + + private readonly ulong[] m_values; + private readonly ulong[] m_marked; + private long m_markTimestamp = Stopwatch.GetTimestamp(); + } + + /// + /// A server exposing only the counter node manager in addition to the standard nodes. + /// + internal sealed class CounterServer : StandardServer + { + public CounterServer(ITelemetryContext telemetry, int count) + : base(telemetry) + { + m_count = count; + } + + public CounterNodeManager? Counters { get; private set; } + + protected override IMasterNodeManager CreateMasterNodeManager( + IServerInternal server, + ApplicationConfiguration configuration) + { + Counters = new CounterNodeManager(server, configuration, m_count); + return new MasterNodeManager(server, configuration, null, Counters); + } + + private readonly int m_count; + } +} diff --git a/perf/ServerLoadHarness/CounterWorkload.cs b/perf/ServerLoadHarness/CounterWorkload.cs new file mode 100644 index 0000000000..a4f6bbd7cb --- /dev/null +++ b/perf/ServerLoadHarness/CounterWorkload.cs @@ -0,0 +1,188 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Client; + +namespace Opc.Ua.Perf.ServerLoadHarness +{ + /// + /// Client observer of the counter workload: one subscription with a monitored item per + /// counter (queue size one, discard oldest). A counter advances once per server sample, + /// so its progress over the window, against window / sampling interval, is the share of + /// samples the server took. Every item must reach 99% to pass, as in the o6-automation + /// opcua-benchmarks subscription suite. + /// + internal sealed class CounterWorkload : Workload + { + public CounterWorkload(ISession session, int count, int samplingInterval, int publishingInterval) + { + m_session = session; + m_samplingInterval = samplingInterval; + m_publishingInterval = publishingInterval; + m_latestValue = new ulong[count]; + m_latestTicks = new long[count]; + m_startValue = new ulong[count]; + m_startTicks = new long[count]; + } + + public override double Operations => Interlocked.Read(ref m_notifications); + + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + ushort namespaceIndex = (ushort)m_session.NamespaceUris.GetIndex(CounterNodeManager.Namespace); +#pragma warning disable CA2000 // ownership transfers to the session in AddSubscription; disposed below on failure before that + var subscription = new Subscription(m_session.DefaultSubscription) + { + PublishingInterval = m_publishingInterval, + KeepAliveCount = 10, + LifetimeCount = 10000, + MaxNotificationsPerPublish = 0, + TimestampsToReturn = TimestampsToReturn.Neither, + FastDataChangeCallback = OnDataChange + }; +#pragma warning restore CA2000 + try + { + for (int i = 0; i < m_latestValue.Length; i++) + { + var item = new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = new NodeId((uint)i + 1, namespaceIndex), + AttributeId = Attributes.Value, + MonitoringMode = MonitoringMode.Reporting, + SamplingInterval = m_samplingInterval, + QueueSize = 1, + DiscardOldest = true + }; + m_indexByHandle[item.ClientHandle] = i; + subscription.AddItem(item); + } + // the session owns and disposes the subscription from here on + m_session.AddSubscription(subscription); + } + catch + { + subscription.Dispose(); + throw; + } + var setup = Stopwatch.StartNew(); + await subscription.CreateAsync().ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"counters: {m_latestValue.Length} items, sampling {m_samplingInterval} ms, ") + + FormattableString.Invariant( + $"publishing {m_publishingInterval} ms, revised sampling {System.Linq.Enumerable.FirstOrDefault(subscription.MonitoredItems)?.Status.SamplingInterval}; setup {setup.ElapsedMilliseconds} ms")); + } + + public override async Task RunAsync(CancellationToken ct) + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + + public override void ResetStats() + { + lock (m_lock) + { + Array.Copy(m_latestValue, m_startValue, m_latestValue.Length); + Array.Copy(m_latestTicks, m_startTicks, m_latestTicks.Length); + } + Interlocked.Exchange(ref m_notifications, 0); + } + + public override string Report(double seconds) + { + double min = double.MaxValue; + double sum = 0; + int below = 0; + int missing = 0; + lock (m_lock) + { + for (int i = 0; i < m_latestValue.Length; i++) + { + if (m_startTicks[i] == 0 || m_latestTicks[i] <= m_startTicks[i]) + { + missing++; + min = 0; + continue; + } + double elapsedMs = (m_latestTicks[i] - m_startTicks[i]) * 1000.0 / Stopwatch.Frequency; + double progress = (m_latestValue[i] - m_startValue[i]) / (elapsedMs / m_samplingInterval); + min = Math.Min(min, progress); + sum += progress; + if (progress < 0.99) + { + below++; + } + } + } + int measured = m_latestValue.Length - missing; + return FormattableString.Invariant( + $"notif/s={Interlocked.Read(ref m_notifications) / seconds:F0} progressMin={min:P2} ") + + FormattableString.Invariant( + $"progressMean={(measured == 0 ? 0 : sum / measured):P2} itemsBelow99={below} itemsWithoutData={missing} ") + + (below == 0 && missing == 0 ? "PASS" : "FAIL"); + } + + private void OnDataChange(Subscription subscription, DataChangeNotification notification, ArrayOf stringTable) + { + long now = Stopwatch.GetTimestamp(); + lock (m_lock) + { + foreach (MonitoredItemNotification item in notification.MonitoredItems) + { + if (m_indexByHandle.TryGetValue(item.ClientHandle, out int index) && + item.Value.WrappedValue.TryGetValue(out ulong value)) + { + m_latestValue[index] = value; + m_latestTicks[index] = now; + } + } + } + Interlocked.Add(ref m_notifications, notification.MonitoredItems.Count); + } + + private readonly ISession m_session; + private readonly int m_samplingInterval; + private readonly int m_publishingInterval; + private readonly Dictionary m_indexByHandle = []; + private readonly ulong[] m_latestValue; + private readonly long[] m_latestTicks; + private readonly ulong[] m_startValue; + private readonly long[] m_startTicks; + private readonly Lock m_lock = new(); + private long m_notifications; + } +} diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index 320d3807ba..b7731f46f7 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -177,6 +177,21 @@ private static async Task RunServerAsync(ApplicationInstance application, I config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); config.ServerConfiguration.MinRequestThreadCount = 50; + // sized for the largest pipeline (sessions x inflight), as the o6 benchmarks do + config.ServerConfiguration.MaxQueuedRequestCount = ArgInt("queue", 2000); + int counters = ArgInt("counters", 0); + int sampling = ArgInt("sampling", 10); + if (counters > 0) + { + // the subscription capacity workload of o6-automation/opcua-benchmarks + config.ServerConfiguration.MinPublishingInterval = 1; + config.ServerConfiguration.PublishingResolution = 1; + config.ServerConfiguration.MaxPublishingInterval = 86400_000; + config.ServerConfiguration.MaxNotificationQueueSize = 1; + config.ServerConfiguration.MaxNotificationsPerPublish = int.MaxValue; + config.ServerConfiguration.MaxPublishRequestCount = 4; + config.ServerConfiguration.AvailableSamplingRates = [new SamplingRateGroup(sampling, 1, 0)]; + } if (!await application .CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime) @@ -186,10 +201,10 @@ private static async Task RunServerAsync(ApplicationInstance application, I return 1; } - using var server = new ReferenceServer(telemetry) - { - TransportBindings = TestTransportBindings.WithAllSchemes() - }; + using Opc.Ua.Server.StandardServer server = counters > 0 + ? new CounterServer(telemetry, counters) + : new ReferenceServer(telemetry); + server.TransportBindings = TestTransportBindings.WithAllSchemes(); await application.StartAsync(server).ConfigureAwait(false); Console.WriteLine(FormattableString.Invariant($"READY {endpointUrl} {Environment.ProcessId}")); @@ -203,10 +218,12 @@ private static async Task RunServerAsync(ApplicationInstance application, I GC.Collect(); GC.WaitForPendingFinalizers(); mark = ServerStats.Capture(); + (server as CounterServer)?.Counters?.Mark(); Console.WriteLine("MARKED"); break; case "report": - Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + string counterReport = (server as CounterServer)?.Counters?.Report(sampling) ?? string.Empty; + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark) + " " + counterReport); break; case "quit": await server.StopAsync().ConfigureAwait(false); @@ -228,7 +245,7 @@ private static async Task RunDriverAsync(string[] args) var startInfo = new ProcessStartInfo( Environment.ProcessPath!, FormattableString.Invariant( - $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}")) + $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")} --counters {(scenario == "counters" ? Arg("nodes", "1") : "0")} --sampling {Arg("sampling", "10")} --queue {Arg("queue", "2000")}")) { RedirectStandardInput = true, RedirectStandardOutput = true, @@ -290,7 +307,9 @@ await Parallel.ForAsync( Console.WriteLine(FormattableString.Invariant( $"connected {sessionCount} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); - Workload workload = scenario == "sub" + Workload workload = scenario == "counters" + ? new CounterWorkload(sessions[0], ArgInt("nodes", 1), ArgInt("sampling", 10), ArgInt("pub", 1000)) + : scenario == "sub" ? new SubscribeWorkload( sessions, ArgInt("nodes", 100), From 6faf39ef19b5a11d6d57572b54ef21a509e81d9e Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:52:28 +0200 Subject: [PATCH 11/16] Reference server: fix wording of the Server GC comment Co-Authored-By: Claude Opus 5.5 --- .../ConsoleReferenceServer/ConsoleReferenceServer.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj index 1f36086530..6bf132e7ce 100644 --- a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj +++ b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj @@ -13,7 +13,7 @@ true true From 086554371674f9576b4357d2899e4a788705e34b Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:55:57 +0200 Subject: [PATCH 12/16] Server: apply backpressure instead of faulting a channel with a full write queue A server channel was faulted with BadTcpNotEnoughResources ("The channel write queue is full") when a request arrived while more than 100 responses were queued for writing, so a client pipelining more requests than that per channel (e.g. the 1000 outstanding Reads per client of the o6-automation benchmark) lost its channel. Server channels now stop reading while the queue is full and resume once it has drained to 50; TCP flow control slows the client instead. Requests already read when the queue filled stay bounded by the request queue limits. Client channels keep reading, because responses are what drain their writes. 10 clients x 1000 outstanding Reads: channels faulted before, now 0 errors at about 45-54k req/s. Co-Authored-By: Claude Opus 5.5 --- src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs | 23 ++--- .../Stack/Tcp/UaSCBinaryChannel.cs | 85 ++++++++++++++++++- .../PipelinedRequestTests.cs | 46 +++++++++- 3 files changed, 134 insertions(+), 20 deletions(-) diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs index f192694225..f2d0dc81e0 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs @@ -45,6 +45,9 @@ namespace Opc.Ua.Bindings /// public class TcpServerChannel : TcpListenerChannel { + /// + protected override bool AppliesReceiveBackpressure => true; + /// /// Attaches the object to an existing socket. /// @@ -1586,15 +1589,9 @@ private bool ProcessRequestMessage( return false; } - if (ChannelFull) - { - m_logger.TcpServerLog13(Id); - m_logger.TcpServerLog14(Id); - ForceChannelFaultCore( - StatusCodes.BadTcpNotEnoughResources, - "The channel write queue is full."); - return false; - } + // A full write queue no longer faults the channel: the receive loop stops + // reading until responses drain (see AppliesReceiveBackpressure). Requests + // already read when the queue filled are bounded by the request queue limits. BufferCollection? chunksToProcess = null; @@ -2105,14 +2102,6 @@ public static partial void TcpServerLog12( uint currentToken, uint previousToken); - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 13, Level = LogLevel.Information, - Message = "Channel {Id}: full -- delay processing.")] - public static partial void TcpServerLog13(this ILogger logger, uint id); - - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 14, Level = LogLevel.Warning, - Message = "Channel {Id}: break socket connection.")] - public static partial void TcpServerLog14(this ILogger logger, uint id); - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 15, Level = LogLevel.Warning, Message = "ChannelId {Id}: ProcessRequestMessage RequestId {RequestId} was aborted.")] public static partial void TcpServerLog15( diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs index adf4af4461..a73f8375c5 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs @@ -965,7 +965,84 @@ private static void DisposeIsolationLeases(List? leases) } /// - public virtual bool ChannelFull => m_activeWriteRequests > 100; + public virtual bool ChannelFull => Volatile.Read(ref m_activeWriteRequests) > 100; + + /// + /// Whether the receive loop stops reading while the write queue is full. + /// + /// + /// A server channel only writes responses to requests it read, so not reading + /// lets TCP flow control slow a client that pipelines more requests than the + /// responses it consumes. A client channel must keep reading: its writes are + /// requests, and responses are what drains them. + /// + protected virtual bool AppliesReceiveBackpressure => false; + + /// + /// Completes when the write queue has room again, or right away when it has. + /// + private ValueTask WaitForWriteCapacityAsync(CancellationToken ct) + { + return AppliesReceiveBackpressure && ChannelFull + ? WaitForWriteCapacitySlowAsync(ct) + : default; + } + + private async ValueTask WaitForWriteCapacitySlowAsync(CancellationToken ct) + { + // resume only once the queue has drained to the low-water mark, so the loop + // reads a batch per wake-up instead of one request per completed write. + while (!HasWriteCapacityToResume) + { + TaskCompletionSource waiter; + lock (m_writeCapacityLock) + { + waiter = m_writeCapacityWaiter ??= + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + // re-checked after publishing the waiter: a write that completed in + // between saw no waiter and did not signal. + if (HasWriteCapacityToResume) + { + return; + } + + using (ct.Register(static state => ((TaskCompletionSource)state!).TrySetCanceled(), waiter)) + { + await waiter.Task.ConfigureAwait(false); + } + } + } + + /// + /// Whether a receive loop paused by a full write queue may read again. + /// + private bool HasWriteCapacityToResume => + !ChannelFull && Volatile.Read(ref m_activeWriteRequests) <= kResumeWriteCount; + + /// + /// The queued-write count a paused receive loop resumes at (half the full mark). + /// + private const int kResumeWriteCount = 50; + + /// + /// Releases a receive loop that waits for write capacity once the queue has room. + /// + private void SignalWriteCapacity() + { + if (Volatile.Read(ref m_writeCapacityWaiter) == null || !HasWriteCapacityToResume) + { + return; + } + TaskCompletionSource? waiter; + lock (m_writeCapacityLock) + { + waiter = m_writeCapacityWaiter; + m_writeCapacityWaiter = null; + } + waiter?.TrySetResult(true); + } /// /// Indicates that admission cleanup must preserve the channel until outstanding writes finish. @@ -1253,6 +1330,9 @@ private async Task RunReceiveLoopAsync(IUaSCByteTransport transport, Cancellatio ArraySegment chunk; try { + // backpressure: do not read more requests while their responses + // cannot be written; the peer's sends block in TCP flow control. + await WaitForWriteCapacityAsync(ct).ConfigureAwait(false); chunk = await transport.ReceiveChunkAsync(ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) @@ -1586,6 +1666,7 @@ protected virtual void HandleWriteComplete( buffers?.Release(BufferManager, "WriteOperation"); Interlocked.Decrement(ref m_activeWriteRequests); + SignalWriteCapacity(); } /// @@ -1979,6 +2060,8 @@ public void UpdateLastActiveTime() /// private int m_state; private int m_activeWriteRequests; + private readonly Lock m_writeCapacityLock = new(); + private TaskCompletionSource? m_writeCapacityWaiter; private readonly Lock m_writeChainLock = new(); private Task m_writeChain = Task.CompletedTask; private long m_lastActiveTimestamp; diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs index 145a362137..33d2060309 100644 --- a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -52,8 +52,50 @@ public class PipelinedRequestTests : TestFixture protected override void ConfigureServer(ApplicationConfiguration configuration) { base.ConfigureServer(configuration); - // admit the whole pipeline: 10 sessions x 90 requests - configuration.ServerConfiguration!.MaxQueuedRequestCount = 2000; + // admit the whole pipeline: up to 10 sessions x 300 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 4000; + } + + /// + /// More requests in flight on a session than the channel's write queue holds (100) + /// slow the client down instead of closing its channel: the server stops reading + /// while the responses drain. The channel used to be faulted with + /// BadTcpNotEnoughResources ("The channel write queue is full"). + /// + [Test] + public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() + { + ArrayOf nodesToRead = new[] + { + new ReadValueId { NodeId = VariableIds.Server_ServerStatus_CurrentTime, AttributeId = Attributes.Value } + }; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var sessions = new List { Session }; + try + { + for (int ii = 1; ii < 4; ii++) + { + sessions.Add(await ClientFixture.ConnectAsync(ServerUrl, SecurityPolicies.None).ConfigureAwait(false)); + } + + for (int round = 0; round < 5; round++) + { + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 300).Select( + _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); + ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); + Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); + } + + Assert.That(sessions.All(session => session.Connected), Is.True); + } + finally + { + foreach (ISession session in sessions.Skip(1)) + { + await session.CloseAsync(5000, true).ConfigureAwait(false); + session.Dispose(); + } + } } /// From 191cd2ba787258c4c164ff9ea213274a45194d28 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:55:57 +0200 Subject: [PATCH 13/16] perf: report client CPU for the measurement window Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/Program.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index b7731f46f7..eda23c12d9 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -333,10 +333,14 @@ await Parallel.ForAsync( Take(serverLines, "MARKED"); workload.ResetStats(); var window = Stopwatch.StartNew(); + TimeSpan clientCpuStart = Process.GetCurrentProcess().TotalProcessorTime; await Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ConfigureAwait(false); await server.StandardInput.WriteLineAsync("report").ConfigureAwait(false); string report = Take(serverLines, "REPORT")[7..]; - string clientReport = workload.Report(window.Elapsed.TotalSeconds); + double clientCores = (Process.GetCurrentProcess().TotalProcessorTime - clientCpuStart).TotalSeconds / + window.Elapsed.TotalSeconds; + string clientReport = workload.Report(window.Elapsed.TotalSeconds) + + FormattableString.Invariant($" clientCpuCores={clientCores:F2}"); await stop.CancelAsync().ConfigureAwait(false); try { From 10292b1eb307d3492f2ac12dd87fd5747320773a Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 10:40:15 +0200 Subject: [PATCH 14/16] Tests: run the deep-pipeline test with 1000 Reads per session In process the client drains responses fast enough that the server's write queue rarely fills, so the test checks that deep pipelines complete with the channels connected; the out-of-process harness exercises the full queue. Co-Authored-By: Claude Opus 5.5 --- .../Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs index 33d2060309..4c5da45c56 100644 --- a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -52,15 +52,17 @@ public class PipelinedRequestTests : TestFixture protected override void ConfigureServer(ApplicationConfiguration configuration) { base.ConfigureServer(configuration); - // admit the whole pipeline: up to 10 sessions x 300 requests - configuration.ServerConfiguration!.MaxQueuedRequestCount = 4000; + // admit the whole pipeline: up to 4 sessions x 1000 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 10000; } /// /// More requests in flight on a session than the channel's write queue holds (100) - /// slow the client down instead of closing its channel: the server stops reading - /// while the responses drain. The channel used to be faulted with - /// BadTcpNotEnoughResources ("The channel write queue is full"). + /// complete and leave the channel connected. When responses back up, the server + /// stops reading until they drain instead of faulting the channel with + /// BadTcpNotEnoughResources ("The channel write queue is full"). In process the + /// client reads fast enough that the queue rarely fills; the out-of-process load + /// harness (perf/ServerLoadHarness, 10 clients x 1000 Reads) exercises the full queue. /// [Test] public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() @@ -80,7 +82,7 @@ public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() for (int round = 0; round < 5; round++) { - IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 300).Select( + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 1000).Select( _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); From d8bdd0295b999aded0601da6ee6f5c7cf7b778e5 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 10:55:04 +0200 Subject: [PATCH 15/16] Server: test the sampling schedule deterministically; tolerate 500 ms of lag SamplingKeepsTheRequestedRateAsync measured the sampling rate against the wall clock and failed on a loaded macOS CI runner (78 % of the samples). The deadline arithmetic now lives in SamplingSchedule, and the tests drive it with simulated clocks: Windows- and Linux-like timer overshoot keeps the rate (where waiting a fixed interval drifts), a 200 ms pause is made up, a 3 s stall restarts the schedule once, and sampling slower than the interval does not run away. The schedule now tolerates a lag of four intervals or 500 ms (was 50 ms) before restarting, so a garbage collection or scheduling pause of that length is made up rather than lost; on a loaded machine this kept more samples at 4096 items (81 % vs 62-71 %). Co-Authored-By: Claude Opus 5.5 --- .../MonitoredItem/SamplingGroup.cs | 19 +- .../MonitoredItem/SamplingSchedule.cs | 109 +++++++++++ .../SamplingGroupLoopRegressionTests.cs | 169 ++++++++++++------ 3 files changed, 225 insertions(+), 72 deletions(-) create mode 100644 src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 287e5a7ef4..5d0750ee8c 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -571,18 +571,8 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance int sleepCycle = Convert.ToInt32(samplingInterval, CultureInfo.InvariantCulture); - // Fixed-rate schedule: each sample has an absolute deadline one interval after - // the previous one. Waiting a fixed interval after each sample instead adds the - // timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on - // Windows) and the sampling time to every cycle, so a 10 ms group sampled about - // 80 to 90 times per second. A late wake-up shortens the next wait, which keeps - // the average rate at the requested interval. long frequency = m_timeProvider.TimestampFrequency; - long period = Math.Max(1, (long)(samplingInterval * frequency / 1000.0)); - // Missed deadlines are made up by sampling again right away, but not after a - // stall longer than this: the schedule restarts instead of bursting. - long maxLag = Math.Max(4 * period, frequency / 20); - long deadline = m_timeProvider.GetTimestamp() + period; + var schedule = SamplingSchedule.Create(samplingInterval, frequency, m_timeProvider.GetTimestamp()); while (m_server.IsRunning && !cancellationToken.IsCancellationRequested) { @@ -591,7 +581,7 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance // wait till next sample without holding a thread. try { - long remaining = deadline - startTimestamp; + long remaining = schedule.GetWait(startTimestamp); if (remaining > 0) { await Task.Delay( @@ -654,14 +644,11 @@ await Task.Delay( break; } - deadline += period; - long now = m_timeProvider.GetTimestamp(); - if (now - deadline > maxLag) + if (schedule.Advance(m_timeProvider.GetTimestamp())) { // the samples take longer than the interval, or the process stalled int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); - deadline = now + period; } } diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs new file mode 100644 index 0000000000..16d0ca87b7 --- /dev/null +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs @@ -0,0 +1,109 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; + +namespace Opc.Ua.Server +{ + /// + /// The fixed-rate schedule of a sampling group, in timestamp ticks. + /// + /// + /// Each sample has an absolute deadline one interval after the previous one. Waiting a + /// fixed interval after each sample instead adds the timer overshoot (about 1 ms on + /// Linux, up to the 15.6 ms timer resolution on Windows) and the sampling time to every + /// cycle, so a 10 ms group sampled about 63 to 90 times per second. With deadlines a + /// late wake-up shortens the next wait, which keeps the average rate at the requested + /// interval. Missed deadlines are made up by sampling again right away, but not after a + /// stall longer than : the schedule then restarts from the current + /// time instead of bursting. + /// + internal struct SamplingSchedule + { + /// + /// Creates a schedule whose first sample is one period after . + /// + public SamplingSchedule(long now, long period, long maxLag) + { + Period = Math.Max(1, period); + MaxLag = maxLag; + Deadline = now + Period; + } + + /// + /// The ticks between two samples. + /// + public long Period { get; } + + /// + /// How far the schedule may fall behind before it restarts. + /// + public long MaxLag { get; } + + /// + /// The timestamp the next sample is due at. + /// + public long Deadline { get; private set; } + + /// + /// Creates the schedule of a sampling interval in milliseconds. It tolerates a lag of + /// four intervals or half a second, whichever is longer, so a garbage collection or + /// scheduling pause of that length is made up rather than lost. + /// + public static SamplingSchedule Create(double samplingIntervalMs, long frequency, long now) + { + long period = Math.Max(1, (long)(samplingIntervalMs * frequency / 1000.0)); + return new SamplingSchedule(now, period, Math.Max(4 * period, frequency / 2)); + } + + /// + /// The ticks to wait before the next sample; zero or less when it is due. + /// + public readonly long GetWait(long now) + { + return Deadline - now; + } + + /// + /// Moves to the deadline after the sample just taken. + /// + /// true when the schedule had fallen too far behind and restarted. + public bool Advance(long now) + { + Deadline += Period; + if (now - Deadline > MaxLag) + { + Deadline = now + Period; + return true; + } + return false; + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index e6172e1415..a1968ddf3c 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -431,69 +431,126 @@ public async Task CustomItemIsSampledImmediatelyOnlyWithoutQueuedInitialValueAsy } /// - /// The loop samples at the requested rate on average. A loop that waits one interval - /// after each sample adds the timer overshoot and the sampling time to every cycle and - /// took 63 % (Windows, 15.6 ms timer) to 90 % (Linux) of the samples of a 10 ms group. + /// A timer that wakes up late on every wait does not slow the schedule down: with a + /// Windows-like 5.6 ms overshoot on a 10 ms interval the schedule still takes a sample + /// every 10 ms on average, where waiting one interval after each sample takes 64 %. /// - [Test] - [NonParallelizable] - public async Task SamplingKeepsTheRequestedRateAsync() + [TestCase(5.6)] + [TestCase(1.0)] + [TestCase(0.0)] + public void ScheduleKeepsTheRateWhenTheTimerOvershoots(double overshootMs) { - Mock server = DeterministicServerMock.Create( - out MonitoredItemQueueFactory queueFactory); - using (queueFactory) + int samples = SimulateSchedule(10, overshootMs, sampleCostMs: 0.3, seconds: 10, out int restarts); + int fixedDelaySamples = SimulateFixedDelay(10, overshootMs, sampleCostMs: 0.3, seconds: 10); + + Assert.That(samples, Is.InRange(995, 1001)); + Assert.That(restarts, Is.Zero); + if (overshootMs > 0) { - int reads = 0; - var nodeManager = new Mock(); - nodeManager - .Setup(m => m.ReadAsync( - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>(), - It.IsAny>(), - It.IsAny())) - .Returns, IList, - IList, CancellationToken>( - (_, _, _, values, _, _) => - { - Interlocked.Increment(ref reads); - for (int ii = 0; ii < values.Count; ii++) - { - values[ii] = new DataValue(new Variant(1)); - } - return default; - }); - nodeManager - .Setup(m => m.ValidateRolePermissionsAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(new ValueTask(ServiceResult.Good)); + Assert.That(fixedDelaySamples, Is.LessThan(975), "the simulation must reproduce the drift"); + } + } - using OperationContext context = CreateContext(); - using var group = new SamplingGroup( - server.Object, nodeManager.Object, [new SamplingRateGroup(10, 1, 0)], context, 10); - Mock item = CreateItem(1, 10); - Assert.That(group.StartMonitoring(context, item.Object), Is.True); - group.ApplyChanges(); + /// + /// A pause shorter than the tolerated lag, such as a garbage collection, is made up by + /// sampling again right away, so no sample of the interval is lost. + /// + [Test] + public void SchedulePausesShorterThanTheLagAreMadeUp() + { + int samples = SimulateSchedule(10, 1.0, 0.3, 10, out int restarts, stallAtMs: 5000, stallMs: 200); - // let the loop start, then count the samples of a fixed window - await Task.Delay(300).ConfigureAwait(false); - int start = Volatile.Read(ref reads); - var window = System.Diagnostics.Stopwatch.StartNew(); - await Task.Delay(2000).ConfigureAwait(false); - int samples = Volatile.Read(ref reads) - start; - double expected = window.Elapsed.TotalMilliseconds / 10; + Assert.That(samples, Is.InRange(995, 1001)); + Assert.That(restarts, Is.Zero); + } - group.StopMonitoring(item.Object); - group.ApplyChanges(); + /// + /// A stall longer than the tolerated lag restarts the schedule once instead of bursting + /// through every missed sample. + /// + [Test] + public void ScheduleRestartsAfterAStallLongerThanTheLag() + { + int samples = SimulateSchedule(10, 1.0, 0.3, 10, out int restarts, stallAtMs: 5000, stallMs: 3000); + + Assert.That(restarts, Is.EqualTo(1)); + Assert.That(samples, Is.InRange(690, 710), "the 3 s stall is skipped, not made up"); + } + + /// + /// Samples that take longer than the interval run back to back without the schedule + /// running away: it restarts whenever it falls a full lag behind. + /// + [Test] + public void ScheduleDoesNotRunAwayWhenSamplingIsSlowerThanTheInterval() + { + int samples = SimulateSchedule(10, 0.0, 15, 10, out int restarts); + + Assert.That(samples, Is.InRange(660, 670)); + Assert.That(restarts, Is.GreaterThan(0)); + } + + private const long kTicksPerSecond = 10_000_000; + + private static long Ticks(double milliseconds) + { + return (long)(milliseconds * kTicksPerSecond / 1000); + } + + private static int SimulateSchedule( + double intervalMs, + double overshootMs, + double sampleCostMs, + double seconds, + out int restarts, + double stallAtMs = -1, + double stallMs = 0) + { + long now = 0; + long end = Ticks(seconds * 1000); + bool stalled = false; + var schedule = SamplingSchedule.Create(intervalMs, kTicksPerSecond, now); + int samples = 0; + restarts = 0; + while (true) + { + long wait = schedule.GetWait(now); + if (wait > 0) + { + now += wait + Ticks(overshootMs); + } + if (!stalled && stallAtMs >= 0 && now >= Ticks(stallAtMs)) + { + stalled = true; + now += Ticks(stallMs); + } + if (now >= end) + { + return samples; + } + now += Ticks(sampleCostMs); + samples++; + if (schedule.Advance(now)) + { + restarts++; + } + } + } - Assert.That(samples / expected, Is.GreaterThanOrEqualTo(0.95), - $"{samples} samples in {window.Elapsed.TotalMilliseconds:F0} ms at a 10 ms interval"); - Assert.That(samples / expected, Is.LessThanOrEqualTo(1.05), - "the loop must not sample faster than requested"); + private static int SimulateFixedDelay(double intervalMs, double overshootMs, double sampleCostMs, double seconds) + { + long now = 0; + long end = Ticks(seconds * 1000); + int samples = 0; + while (true) + { + now += Ticks(intervalMs + overshootMs); + if (now >= end) + { + return samples; + } + now += Ticks(sampleCostMs); + samples++; } } From f2df124124024c3cc1974e67bd6b76a86571c19c Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 11:10:57 +0200 Subject: [PATCH 16/16] Fix sampling delay overflow for long intervals; await snapshot test readers The wait before a sample was converted with remaining * TicksPerSecond / frequency, which overflows a long for long intervals at the 1 GHz timestamp frequency of Unix (one hour is 3.6e19); Task.Delay then threw and ended the sampling loop. SamplingSchedule.ToTimeSpan scales through a double, covered by a one-hour test at 1 GHz. ResourceManagerSnapshotTests awaits its readers with a timeout instead of blocking on Task.WaitAll. Co-Authored-By: Claude Opus 5.5 --- .../NodeManager/MonitoredItem/SamplingGroup.cs | 2 +- .../NodeManager/MonitoredItem/SamplingSchedule.cs | 12 ++++++++++++ .../SamplingGroupLoopRegressionTests.cs | 15 +++++++++++++++ .../ResourceManagerSnapshotTests.cs | 7 ++++--- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 5d0750ee8c..fee3983f3a 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -585,7 +585,7 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance if (remaining > 0) { await Task.Delay( - TimeSpan.FromTicks(remaining * TimeSpan.TicksPerSecond / frequency), + SamplingSchedule.ToTimeSpan(remaining, frequency), cancellationToken).ConfigureAwait(false); } else diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs index 16d0ca87b7..3bd27f40e5 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs @@ -83,6 +83,18 @@ public static SamplingSchedule Create(double samplingIntervalMs, long frequency, return new SamplingSchedule(now, period, Math.Max(4 * period, frequency / 2)); } + /// + /// Converts a span of timestamp ticks to a . + /// + /// + /// Scaled through a double: multiplying first overflows a long for long intervals at + /// a 1 GHz timestamp frequency (Unix), e.g. one hour is 3.6e12 ticks times 1e7. + /// + public static TimeSpan ToTimeSpan(long timestampTicks, long frequency) + { + return TimeSpan.FromTicks((long)(timestampTicks * ((double)TimeSpan.TicksPerSecond / frequency))); + } + /// /// The ticks to wait before the next sample; zero or less when it is due. /// diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index a1968ddf3c..c42fe7de08 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -490,6 +490,21 @@ public void ScheduleDoesNotRunAwayWhenSamplingIsSlowerThanTheInterval() Assert.That(restarts, Is.GreaterThan(0)); } + /// + /// Long waits convert to a delay without overflowing at the 1 GHz timestamp frequency + /// of Unix: the default one-hour sampling group would otherwise compute 3.6e19 ticks. + /// + [Test] + public void ScheduleWaitOfAnHourConvertsWithoutOverflowAtOneGigahertz() + { + const long frequency = 1_000_000_000; + var schedule = SamplingSchedule.Create(3_600_000, frequency, 0); + + TimeSpan wait = SamplingSchedule.ToTimeSpan(schedule.GetWait(0), frequency); + + Assert.That(wait, Is.EqualTo(TimeSpan.FromHours(1))); + } + private const long kTicksPerSecond = 10_000_000; private static long Ticks(double milliseconds) diff --git a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs index 7b0d53add2..0dac8f475d 100644 --- a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs +++ b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs @@ -28,6 +28,7 @@ * ======================================================================*/ +using System; using System.Globalization; using System.Threading; using System.Threading.Tasks; @@ -73,7 +74,7 @@ public void UpdatesAndNewLocalesAreVisibleAfterASnapshotWasBuilt() /// written table, and see every translation once the writer is done. /// [Test] - public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() + public async Task ConcurrentReadersSeeACompleteSnapshotOnceWritesCompleteAsync() { using ResourceManager resourceManager = CreateResourceManager(); const int keyCount = 500; @@ -108,8 +109,8 @@ public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() "de-DE", "Wert" + ii.ToString(CultureInfo.InvariantCulture)); } - cts.Cancel(); - Assert.That(Task.WaitAll(readers, 30000), Is.True); + await cts.CancelAsync().ConfigureAwait(false); + await Task.WhenAll(readers).WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.That(wrongTexts, Is.Zero); for (int ii = 0; ii < keyCount; ii++)