From a858f843896e09ee5dbaaa160b115b4ed2eb3430 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 00:11:56 +0200 Subject: [PATCH 01/19] Server: cut per-request locking and allocation on the request path Profiling a 20-session load (read/write/browse/subscribe) against the reference server showed the request path dominated by lock spinning and per-node allocation. Paired A/B medians (workstation GC, 24 cores): Read 1 value +45% req/s (CPU/req -37%, alloc/req -39%, lock contention -94%), Read 100 values +40%, encrypted Read 100 +48%, Browse +33%, Write 100 +13%, data-change notifications -17% alloc. - ObjectPool: lock-free slot array; ConcurrentBag.Count took every per-thread lock on each Return of the pooled response source. - FairRequestQueue: drop redundant gate entries (fail-fast pre-checks, stopped check, owner count folded into selection) and coalesce wake pulses so the bounded signal channel's lock is only taken when no token is pending. - Isolation provider: compare peer addresses without allocating; build the unsecured verification context only when a classifier is set. - TCP listener: cache the certificate DER copies per channel instead of copying X509 RawData for every request. - EndpointBase: resolve the request ActivitySource once instead of walking the stack (GetCallingAssembly) per request. - Node managers: reuse one ServerSystemContext copy per request in the per-node callbacks, remove a discarded copy, use plain dictionaries for per-request operation caches, build the validation cache in one pass and fix zero-capacity error lists. - ResourceManager: lock-free translation lookups from an immutable snapshot rebuilt after writes (Browse serialised on it). - MonitoredItem: share the default DataChangeFilter instead of allocating one per sampled value. - Reference server: enable Server GC; documented in ServerScalability.md (doubles Read throughput under load vs workstation GC). Co-Authored-By: Claude Opus 5.5 --- docs/ServerScalability.md | 15 +++ .../ConsoleReferenceServer.csproj | 5 + .../EndpointBase.EndpointIncomingRequest.cs | 3 +- src/Opc.Ua.Core/Stack/Server/EndpointBase.cs | 21 ++++ .../Stack/Server/FairRequestQueue.cs | 73 +++++++----- .../Stack/Tcp/TcpTransportListener.cs | 4 +- .../Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs | 35 ++++++ .../Stack/Tcp/UaSCBinaryChannel.cs | 2 +- .../Types/Utils/ValueTask/ObjectPool.cs | 54 +++++++-- .../NodeManager/AsyncCustomNodeManager.cs | 59 ++++------ .../NodeManager/CustomNodeManager.cs | 16 +-- .../NodeManager/CustomNodeManagerAsync.cs | 2 +- .../NodeManagerServiceDispatcher.cs | 17 +-- .../NodeManager/ResourceManager.cs | 111 ++++++++++++------ .../DefaultServerResourceIsolationProvider.cs | 58 ++++++--- src/Opc.Ua.Server/Server/OperationContext.cs | 40 +++++++ .../MonitoredItem/MonitoredItem.cs | 18 ++- 17 files changed, 374 insertions(+), 159 deletions(-) diff --git a/docs/ServerScalability.md b/docs/ServerScalability.md index b4b4f59236..53dc4c6095 100644 --- a/docs/ServerScalability.md +++ b/docs/ServerScalability.md @@ -9,6 +9,7 @@ This document describes how the reference server scales to large numbers of conc - [Admission control and rate limiting](#admission-control-and-rate-limiting) - [Held Publishes and the request-thread budget](#held-publishes-and-the-request-thread-budget) - [Session diagnostics cost](#session-diagnostics-cost) +- [Garbage collector mode](#garbage-collector-mode) - [Configuration](#configuration) - [Scaling out](#scaling-out) - [See also](#see-also) @@ -67,6 +68,20 @@ starving other requests. Creating a session or subscription registers a diagnostics node and marks the live `SessionDiagnostics` and `SubscriptionDiagnostics` arrays for refresh. The arrays are rebuilt on demand when they are read or monitored, and the rebuild is throttled so that a burst of session creates does not trigger a rebuild per create. Servers that do not need the live session-diagnostics arrays can turn them off with the `DiagnosticsEnabled` server setting to remove the cost entirely. +## Garbage collector mode + +A server allocates per request on many threads at once. With the default workstation garbage collector the heap's gen0 budget is a few megabytes, so a loaded server collects hundreds of times per second and every collection suspends all request threads. In a 20-session Read load (100 values per request, 24-core machine) the workstation collector paused the process for about half of the wall-clock time; enabling Server GC on the same build doubled the request rate (21k to 44k Reads per second) and cut the 90th-percentile latency from 2.1 ms to 0.6 ms, at the price of a larger working set (about 430 MB instead of 210 MB). + +Enable Server GC in the server's project file: + +```xml + + true + +``` + +or set `DOTNET_gcServer=1` in the environment. On .NET 8 and later, Server GC adapts its heap count to the load (DATAS), so idle servers stay small. The console reference server enables it. + ## Configuration The following settings size a single node for its hardware (see also the sizing notes in [Benchmarks.md](Benchmarks.md#server-session-scalability) and the admission-control settings in [RateLimiting.md](RateLimiting.md)): diff --git a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj index 5a0bb76678..1f36086530 100644 --- a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj +++ b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj @@ -11,6 +11,11 @@ Quickstarts 46345736-a30f-4466-b3bb-42548ecfaacc true + + true diff --git a/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs b/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs index 86d308794f..e20995acc1 100644 --- a/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs +++ b/src/Opc.Ua.Core/Stack/Server/EndpointBase.EndpointIncomingRequest.cs @@ -109,8 +109,7 @@ public readonly async ValueTask CallAsync(CancellationToken cancellationToken = try { Activity? activity = null; - ActivitySource activitySource = m_endpoint.MessageContext.Telemetry - .GetActivitySource(); + ActivitySource activitySource = m_endpoint.RequestActivitySource; if (activitySource.HasListeners()) { // extract trace information from the request header if available diff --git a/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs b/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs index 40ecf596b3..3c479e9dca 100644 --- a/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs +++ b/src/Opc.Ua.Core/Stack/Server/EndpointBase.cs @@ -518,6 +518,26 @@ internal static ServiceFault CreateFault( /// The message context. protected IServiceMessageContext MessageContext => m_server!.MessageContext; + /// + /// The activity source for incoming requests. Resolved once per telemetry + /// context: the GetActivitySource extension walks the stack to find the + /// calling assembly, which is too expensive to repeat for every request. + /// + private ActivitySource RequestActivitySource + { + get + { + ITelemetryContext telemetry = MessageContext.Telemetry; + Tuple? cached = m_activitySource; + if (cached == null || !ReferenceEquals(cached.Item1, telemetry)) + { + cached = Tuple.Create(telemetry, telemetry.GetActivitySource()); + m_activitySource = cached; + } + return cached.Item2; + } + } + /// /// Returns the description for the endpoint /// @@ -584,6 +604,7 @@ protected delegate ValueTask InvokeService( private IServiceHostBase? m_host; private IServerBase? m_server; + private Tuple? m_activitySource; } /// diff --git a/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs b/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs index 056593e9b6..a735900d5f 100644 --- a/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs +++ b/src/Opc.Ua.Core/Stack/Server/FairRequestQueue.cs @@ -78,16 +78,11 @@ public FairRequestQueue( /// /// Gets the number of requests still waiting for execution admission. /// - internal int Count - { - get - { - lock (m_gate) - { - return m_count; - } - } - } + /// + /// Read without the queue gate: the count is only mutated under the gate and + /// callers use it as a snapshot (worker growth heuristics, diagnostics). + /// + internal int Count => Volatile.Read(ref m_count); /// /// Gets the number of scheduling FIFOs with queued requests: one per owner in weighted @@ -128,18 +123,17 @@ public bool TryEnqueue( error = StatusCodes.BadRequestCancelledByClient; return false; } - lock (m_gate) + // Fail fast without the gate; both conditions are re-checked under the + // gate before the entry is published. + if (Volatile.Read(ref m_stopped)) { - if (m_stopped) - { - error = StatusCodes.BadServerHalted; - return false; - } - if (m_count >= m_maxQueuedRequests) - { - error = StatusCodes.BadServerTooBusy; - return false; - } + error = StatusCodes.BadServerHalted; + return false; + } + if (Volatile.Read(ref m_count) >= m_maxQueuedRequests) + { + error = StatusCodes.BadServerTooBusy; + return false; } ResourceIsolationOwner owner = Classify(request); @@ -225,8 +219,8 @@ public bool TryEnqueue( /// public bool TryDequeue([NotNullWhen(true)] out Entry? result) { - int attempts = OwnerCount; - while (attempts-- > 0 && TrySelect(out Entry? candidate, out long version)) + int attempts = -1; + while (TrySelect(ref attempts, out Entry? candidate, out long version)) { IDisposable? executionLease = null; StatusCode error = StatusCodes.Good; @@ -323,12 +317,9 @@ public async ValueTask DequeueAsync(CancellationToken cancellationToken) while (true) { cancellationToken.ThrowIfCancellationRequested(); - lock (m_gate) + if (Volatile.Read(ref m_stopped)) { - if (m_stopped) - { - throw new OperationCanceledException(cancellationToken); - } + throw new OperationCanceledException(cancellationToken); } if (TryDequeue(out Entry? entry)) { @@ -337,6 +328,10 @@ public async ValueTask DequeueAsync(CancellationToken cancellationToken) try { await m_signals.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + // The token is consumed; let the next Pulse publish a new one. This + // reader dequeues after the reset, so a Pulse skipped before it is + // still observed. + Volatile.Write(ref m_signalPending, 0); } catch (ChannelClosedException) { @@ -434,13 +429,19 @@ private static bool IsControlRequest(IServiceRequest request) /// /// Selects one eligible owner head while excluding concurrent provider admission attempts. + /// The first call of a dequeue pass ( negative) snapshots the + /// number of scheduling FIFOs under the same gate acquisition; each call consumes one attempt. /// - private bool TrySelect([NotNullWhen(true)] out Entry? entry, out long version) + private bool TrySelect(ref int attempts, [NotNullWhen(true)] out Entry? entry, out long version) { lock (m_gate) { version = Volatile.Read(ref m_version); - if (!m_stopped && !m_selecting) + if (attempts < 0) + { + attempts = m_owners.Count; + } + if (attempts-- > 0 && !m_stopped && !m_selecting) { int remaining = m_round.Count; while (remaining-- > 0 && m_round.First != null) @@ -539,9 +540,16 @@ private void Changed() /// /// Retains at most one wake token and hands it to only one reader, never broadcasting. /// + /// + /// Pulse runs on every admission, dispatch and capacity release. While a token is + /// already pending the call returns without entering the signal channel's lock, + /// which otherwise serialises every request thread on it. + /// private void Pulse() { if (Volatile.Read(ref m_count) > 0 && !Volatile.Read(ref m_stopped) && + Volatile.Read(ref m_signalPending) == 0 && + Interlocked.Exchange(ref m_signalPending, 1) == 0 && m_signals.Writer.TryWrite(true)) { Interlocked.Increment(ref m_wakeSignalCount); @@ -857,6 +865,11 @@ public void Dispose() /// private readonly Channel m_signals = Channel.CreateBounded(1); + /// + /// 1 while a wake token is in and not yet consumed. + /// + private int m_signalPending; + /// /// Number of linked requests. /// diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs index a4c3823e4c..b12a02cd54 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs @@ -1721,8 +1721,8 @@ private async void OnRequestReceivedAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs index 89905d7c23..e0ff3022f8 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs @@ -99,6 +99,20 @@ protected string SecurityPolicyUri /// internal Certificate? ClientCertificate { get; set; } + /// + /// The DER encoding of , copied once per certificate. + /// + /// + /// Certificate.RawData returns a new copy on every access; the listener needs the + /// bytes for every request it dispatches. + /// + internal byte[]? ClientCertificateRawData => GetCachedRawData(ClientCertificate, ref m_clientCertificateRawData); + + /// + /// The DER encoding of , copied once per certificate. + /// + internal byte[]? ServerCertificateRawData => GetCachedRawData(ServerCertificate, ref m_serverCertificateRawData); + /// /// The client certificate chain. /// @@ -2459,5 +2473,26 @@ private static RsaUtils.Padding GetAsymmetricPadding(SecurityPolicyInfo policy) private bool m_noncesDisposed; private Nonce? m_localNonce; private Nonce? m_remoteNonce; + private Tuple? m_clientCertificateRawData; + private Tuple? m_serverCertificateRawData; + + /// + /// Returns the cached DER copy of a certificate, refreshing it when the + /// certificate instance changes (renewal, reassignment). + /// + private static byte[]? GetCachedRawData(Certificate? certificate, ref Tuple? cache) + { + if (certificate == null) + { + return null; + } + Tuple? cached = Volatile.Read(ref cache); + if (cached == null || !ReferenceEquals(cached.Item1, certificate)) + { + cached = Tuple.Create(certificate, certificate.RawData); + Volatile.Write(ref cache, cached); + } + return cached.Item2; + } } } diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs index b8dd806544..adf4af4461 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs @@ -646,7 +646,7 @@ private bool SaveChunk( { var context = new SecureChannelContext( GlobalChannelId, EndpointDescription, RequestEncoding.Binary, - ClientCertificate?.RawData, ServerCertificate?.RawData, ChannelThumbprint, + ClientCertificateRawData, ServerCertificateRawData, ChannelThumbprint, (Transport?.RemoteEndpoint as System.Net.IPEndPoint)?.Address); owner = isolation is IResourceIsolationReassemblyProvider reassembly ? reassembly.ClassifyReassembly(context) : isolation.Classify(context); diff --git a/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs b/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs index 48bc0c53e7..2b0eee3480 100644 --- a/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs +++ b/src/Opc.Ua.Core/Types/Utils/ValueTask/ObjectPool.cs @@ -27,31 +27,45 @@ * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ + using System; -using System.Collections.Concurrent; +using System.Threading; namespace Opc.Ua { /// - /// A simple object pool implementation. + /// A simple bounded, lock-free object pool. /// + /// + /// Get and Return run once per service request on the server hot path and + /// typically on different threads (the request is taken on a receive thread + /// and returned on whichever thread awaits the response). A fixed slot array + /// with interlocked exchange keeps both operations wait-free; a + /// ConcurrentBag would steal across thread-local lists and its Count takes + /// every per-thread lock. + /// /// The type of object to pool. internal class ObjectPool where T : class { - private readonly ConcurrentBag m_objects; private readonly Func m_objectGenerator; - private readonly int m_maxSize; + private readonly T?[] m_items; + private T? m_firstItem; /// /// Initializes a new instance of the class. /// /// The function to generate new objects. - /// The maximum size of the pool. + /// The maximum number of retained objects (capped at four per processor). public ObjectPool(Func objectGenerator, int maxSize) { m_objectGenerator = objectGenerator ?? throw new ArgumentNullException(nameof(objectGenerator)); - m_maxSize = maxSize > 0 ? maxSize : throw new ArgumentOutOfRangeException(nameof(maxSize)); - m_objects = []; + if (maxSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxSize)); + } + // Retained objects only need to cover the concurrently outstanding ones; + // more slots would only lengthen the scan when the pool runs dry. + m_items = new T?[Math.Max(Math.Min(maxSize, Environment.ProcessorCount * 4), 1) - 1]; } /// @@ -60,11 +74,22 @@ public ObjectPool(Func objectGenerator, int maxSize) /// An object from the pool or a new one if the pool is empty. public T Get() { - if (m_objects.TryTake(out T? item)) + T? item = m_firstItem; + if (item != null && Interlocked.CompareExchange(ref m_firstItem, null, item) == item) { return item; } + T?[] items = m_items; + for (int i = 0; i < items.Length; i++) + { + item = items[i]; + if (item != null && Interlocked.CompareExchange(ref items[i], null, item) == item) + { + return item; + } + } + return m_objectGenerator(); } @@ -74,9 +99,18 @@ public T Get() /// The object to return. public void Return(T item) { - if (m_objects.Count < m_maxSize) + if (m_firstItem == null && Interlocked.CompareExchange(ref m_firstItem, item, null) == null) + { + return; + } + + T?[] items = m_items; + for (int i = 0; i < items.Length; i++) { - m_objects.Add(item); + if (Interlocked.CompareExchange(ref items[i], item, null) == null) + { + return; + } } } } diff --git a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs index 8ef00085e0..88cb9791b8 100644 --- a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs @@ -643,6 +643,15 @@ ILocalAddressSpace ILocalAddressSpaceSource.CreateLocalAddressSpace() /// public ServerSystemContext SystemContext { get; } + /// + /// Returns the system context for an operation, reusing the copy already made + /// for the same request. Use it in callbacks that run once per node of a request. + /// + private ServerSystemContext GetOperationSystemContext(OperationContext context) + { + return context != null ? context.GetSystemContext(SystemContext) : SystemContext.Copy(context!); + } + /// /// Gets the default index for the node manager's namespace. /// @@ -4345,7 +4354,7 @@ public async ValueTask GetNodeMetadataAsync( BrowseResultMask resultMask, CancellationToken cancellationToken = default) { - ServerSystemContext systemContext = SystemContext.Copy(context); + ServerSystemContext systemContext = GetOperationSystemContext(context); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(targetHandle); @@ -4716,8 +4725,6 @@ protected async ValueTask GetReferenceDescriptionAsync( ContinuationPoint continuationPoint, CancellationToken cancellationToken = default) { - _ = SystemContext.Copy(context); - // create the type definition reference. var description = new ReferenceDescription { NodeId = reference.TargetId }; description.SetReferenceType( @@ -4821,7 +4828,7 @@ public virtual async ValueTask TranslateBrowsePathAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(sourceHandle); @@ -4944,7 +4951,7 @@ public virtual async ValueTask ReadAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); for (int ii = 0; ii < nodesToRead.Count; ii++) @@ -5208,7 +5215,7 @@ public virtual async ValueTask WriteAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); @@ -5936,7 +5943,7 @@ public virtual async ValueTask HistoryReadAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); for (int ii = 0; ii < nodesToRead.Count; ii++) @@ -6808,7 +6815,7 @@ public virtual async ValueTask HistoryUpdateAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); @@ -7422,7 +7429,7 @@ public virtual async ValueTask FindMethodStateAsync( } ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); NodeHandle? handle = await GetManagerHandleAsync( systemContext, @@ -7482,7 +7489,7 @@ public virtual async ValueTask CallAsync( CancellationToken cancellationToken = default) { ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); for (int ii = 0; ii < methodsToCall.Count; ii++) { @@ -8244,7 +8251,7 @@ public virtual async ValueTask RestoreMonitoredItemsAsync( } ServerSystemContext systemContext = SystemContext.Copy(); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var restoredItems = new List(); @@ -8416,7 +8423,7 @@ public virtual async ValueTask CreateMonitoredItemsAsync( { using NodeManagerOperation nodeOperation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var createdItems = new List(); @@ -10369,7 +10376,7 @@ public virtual ValueTask IsNodeInViewAsync( if (handle.Node != null) { return new ValueTask( - IsNodeInView(SystemContext.Copy(context), viewId, handle.Node)); + IsNodeInView(GetOperationSystemContext(context), viewId, handle.Node)); } return new ValueTask(false); @@ -10390,7 +10397,7 @@ public virtual ValueTask IsNodeInViewAsync( bool permissionsOnly, CancellationToken cancellationToken = default) { - ServerSystemContext systemContext = SystemContext.Copy(context); + ServerSystemContext systemContext = GetOperationSystemContext(context); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(targetHandle); @@ -10408,8 +10415,6 @@ public virtual ValueTask IsNodeInViewAsync( return null!; } - var values = new Variant[3]; - // construct the meta-data object. var metadata = new NodeMetadata(target, target.NodeId) { @@ -10420,25 +10425,10 @@ public virtual ValueTask IsNodeInViewAsync( if (uniqueNodesServiceAttributesCache != null) { NodeId key = handle.NodeId; - if (uniqueNodesServiceAttributesCache.ContainsKey(key)) - { - if (uniqueNodesServiceAttributesCache[key].Length != 3) - { - ReadAndCacheValidationAttributes( - uniqueNodesServiceAttributesCache, - systemContext, - target, - key, - ref values); - } - else - { - // Retrieve value from cache - values = uniqueNodesServiceAttributesCache[key]; - } - } - else + if (!uniqueNodesServiceAttributesCache.TryGetValue(key, out Variant[]? values) || + values.Length != 3) { + values = new Variant[3]; ReadAndCacheValidationAttributes( uniqueNodesServiceAttributesCache, systemContext, @@ -10451,6 +10441,7 @@ public virtual ValueTask IsNodeInViewAsync( } // All other calls that do not use the cache else if (permissionsOnly) { + var values = new Variant[3]; ReadValidationAttributes(systemContext, target, ref values); SetAccessAndRolePermissions(values, metadata); } diff --git a/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs index 98792081e1..b0a8cb599d 100644 --- a/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs @@ -2375,7 +2375,7 @@ public virtual void TranslateBrowsePath( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); // check for valid handle. NodeHandle? handle = IsHandleInNamespace(sourceHandle); @@ -2486,7 +2486,7 @@ public virtual void Read( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); lock (Lock) @@ -2741,7 +2741,7 @@ public virtual void Write( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var nodesToNotify = new List(); @@ -3391,7 +3391,7 @@ public virtual void HistoryRead( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); lock (Lock) @@ -3910,7 +3910,7 @@ public virtual void HistoryUpdate( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToProcess = new List(); lock (Lock) @@ -4420,7 +4420,7 @@ public virtual MethodState FindMethodState( } ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); NodeHandle? handle = GetManagerHandle( systemContext, @@ -4939,7 +4939,7 @@ public virtual void RestoreMonitoredItems( } ServerSystemContext systemContext = SystemContext.Copy(); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); for (int ii = 0; ii < itemsToRestore.Count; ii++) @@ -5098,7 +5098,7 @@ public virtual void CreateMonitoredItems( { using NodeManagerOperation operation = BeginNodeManagerOperation(); ServerSystemContext systemContext = SystemContext.Copy(context); - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); var nodesToValidate = new List(); var createdItems = new List(); diff --git a/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs b/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs index 1b19d5c4bf..2d68511c9b 100644 --- a/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs +++ b/src/Opc.Ua.Server/NodeManager/CustomNodeManagerAsync.cs @@ -103,7 +103,7 @@ private ServiceResult CallSynchronousMethod( ArrayOf methodsToCall, IList errors) { - IDictionary operationCache = new NodeIdDictionary(); + IDictionary operationCache = new Dictionary(); for (int ii = 0; ii < methodsToCall.Count; ii++) { CallMethodRequest request = methodsToCall[ii]; diff --git a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs index 4ca7a05d40..251eeda8d1 100644 --- a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs +++ b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs @@ -714,7 +714,6 @@ private static void PrepareValidationCache( ArrayOf nodesList, out Dictionary uniqueNodesServiceAttributes) { - var uniqueNodes = new HashSet(); Type listType = typeof(T); if (listType != typeof(ReadValueId) && @@ -727,6 +726,8 @@ private static void PrepareValidationCache( nameof(nodesList)); } + // uniqueNodesServiceAttributes is the place where the attributes for each unique nodeId are kept on the services + uniqueNodesServiceAttributes = new Dictionary(nodesList.Count); for (int i = 0; i < nodesList.Count; i++) { NodeId nodeId = default; @@ -746,15 +747,9 @@ private static void PrepareValidationCache( if (!nodeId.IsNull) { - uniqueNodes.Add(nodeId); + uniqueNodesServiceAttributes.TryAdd(nodeId, []); } } - // uniqueNodesReadAttributes is the place where the attributes for each unique nodeId are kept on the services - uniqueNodesServiceAttributes = []; - foreach (NodeId uniqueNode in uniqueNodes) - { - uniqueNodesServiceAttributes.Add(uniqueNode, []); - } } /// @@ -1223,7 +1218,7 @@ private async ValueTask UpdateReferenceDescriptionAsync( var diagnosticInfos = new List(nodesToRead.Count); // create empty list of errors. - var errors = new List(values.Count); + var errors = new List(nodesToRead.Count); for (int ii = 0; ii < nodesToRead.Count; ii++) { errors.Add(null!); @@ -1370,7 +1365,7 @@ await nodeManager.ReadAsync( // pre-validate items. bool validItems = false; // create empty list of errors. - var errors = new List(results.Count); + var errors = new List(nodesToRead.Count); for (int ii = 0; ii < nodesToRead.Count; ii++) { errors.Add(null!); @@ -1709,7 +1704,7 @@ await nodeManager.WriteAsync( bool validItems = false; // create empty list of errors. - var errors = new List(results.Count); + var errors = new List(nodesToUpdate.Count); for (int ii = 0; ii < nodesToUpdate.Count; ii++) { errors.Add(null!); diff --git a/src/Opc.Ua.Server/NodeManager/ResourceManager.cs b/src/Opc.Ua.Server/NodeManager/ResourceManager.cs index 6b5a6e5834..57719f2729 100644 --- a/src/Opc.Ua.Server/NodeManager/ResourceManager.cs +++ b/src/Opc.Ua.Server/NodeManager/ResourceManager.cs @@ -247,6 +247,7 @@ public void Add(string key, string locale, string text) { TranslationTable? table = GetTable(culture.Name); table!.Translations[key] = text; + m_translationSnapshot = null; } } @@ -284,6 +285,7 @@ public void Add(string locale, IDictionary translations) { table!.Translations[translation.Key] = translation.Value; } + m_translationSnapshot = null; } } @@ -406,38 +408,34 @@ protected virtual LocalizedText Translate( : []; #endif // If only mul/qst is requested, return all available translations for the key. + TranslationTable[] tables = GetTranslationSnapshot(); if (preferredLocales.Count == 1) { - lock (m_lock) + foreach (TranslationTable table in tables) { - foreach (TranslationTable table in m_translationTables) + if (table.Translations + .TryGetValue((info.Key ?? info.Text)!, out string? translation)) { - if (table.Translations - .TryGetValue((info.Key ?? info.Text)!, out string? translation)) - { - translations[table!.Locale!.Name] = translation!; - } + translations[table!.Locale!.Name] = translation!; } } } else { // mul/qst + specific locales: return only those translations - lock (m_lock) + for (int i = 1; i < preferredLocales.Count; i++) { - for (int i = 1; i < preferredLocales.Count; i++) + string? translation = FindBestTranslation( + tables, + preferredLocales.Slice(i, 1), + (info.Key ?? info.Text)!, + out CultureInfo culture); + if (translation != null) { - string? translation = FindBestTranslation( - preferredLocales.Slice(i, 1), - (info.Key ?? info.Text)!, - out CultureInfo culture); - if (translation != null) - { - // label the text with the locale of the table it came - // from, which may be another region of the requested - // language. - translations[culture.Name] = translation; - } + // label the text with the locale of the table it came + // from, which may be another region of the requested + // language. + translations[culture.Name] = translation; } } } @@ -453,20 +451,18 @@ protected virtual LocalizedText Translate( string? translatedText = info.Text; CultureInfo culture = CultureInfo.InvariantCulture; - lock (m_lock) - { - translatedText = FindBestTranslation( - preferredLocales, - (info.Key ?? info.Text)!, - out culture); + translatedText = FindBestTranslation( + GetTranslationSnapshot(), + preferredLocales, + (info.Key ?? info.Text)!, + out culture); - // use the default if no translation available. - if (translatedText == null) - { - return preferredLocales.Count > 0 && defaultText.Translations == null && info.Text != null - ? new LocalizedText(info) - : defaultText.FilterByPreferredLocales(preferredLocales); - } + // use the default if no translation available. + if (translatedText == null) + { + return preferredLocales.Count > 0 && defaultText.Translations == null && info.Text != null + ? new LocalizedText(info) + : defaultText.FilterByPreferredLocales(preferredLocales); } // construct translated localized text. @@ -483,6 +479,41 @@ private class TranslationTable public SortedDictionary Translations = []; } + /// + /// Returns an immutable copy of the translation tables. Translations are looked up + /// for every localized text a service returns (each DisplayName of a Browse), while + /// the tables are written almost only at startup, so readers share a snapshot instead + /// of serialising on the lock. The dictionaries are copied, never shared with a writer. + /// + private TranslationTable[] GetTranslationSnapshot() + { + TranslationTable[]? snapshot = Volatile.Read(ref m_translationSnapshot); + if (snapshot != null) + { + return snapshot; + } + + lock (m_lock) + { + snapshot = m_translationSnapshot; + if (snapshot == null) + { + snapshot = new TranslationTable[m_translationTables.Count]; + for (int ii = 0; ii < snapshot.Length; ii++) + { + TranslationTable table = m_translationTables[ii]; + snapshot[ii] = new TranslationTable + { + Locale = table.Locale, + Translations = new SortedDictionary(table.Translations) + }; + } + Volatile.Write(ref m_translationSnapshot, snapshot); + } + return snapshot; + } + } + /// /// Finds the translation table for the locale. Creates a new table if it does not exist. /// @@ -504,6 +535,7 @@ private class TranslationTable // add table. var table = new TranslationTable { Locale = new CultureInfo(locale) }; m_translationTables.Add(table); + m_translationSnapshot = null; return table; } @@ -569,7 +601,8 @@ private static bool ContainsAllLocales( /// /// Finds the best translation for the requested locales. /// - private string? FindBestTranslation( + private static string? FindBestTranslation( + TranslationTable[] tables, ArrayOf preferredLocales, string key, out CultureInfo culture) @@ -602,9 +635,9 @@ private static bool ContainsAllLocales( // search for translation. string? translatedText = null; - for (int ii = 0; ii < m_translationTables.Count; ii++) + for (int ii = 0; ii < tables.Length; ii++) { - TranslationTable translationTable = m_translationTables[ii]; + TranslationTable translationTable = tables[ii]; // all done if exact match found (locale ids are case-insensitive, RFC 5646). if (string.Equals( @@ -706,6 +739,12 @@ private LocalizedText TranslateSymbolicId( private readonly Lock m_lock = new(); private readonly List m_translationTables; + + /// + /// Immutable copy of read without the lock; + /// null after a change until the next translation rebuilds it. + /// + private TranslationTable[]? m_translationSnapshot; private Dictionary? m_statusCodeMapping; private Dictionary? m_symbolicIdMapping; } diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index db3f6881e5..1055942831 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -273,22 +273,25 @@ public StatusCode GetRevalidationStatus( ? StatusCodes.BadSessionIdInvalid : StatusCodes.BadServerTooBusy; } - bool secure = original.Channel.SecurityMode is - MessageSecurityMode.Sign or MessageSecurityMode.SignAndEncrypt && - original.Channel.SecurityPolicyUri != SecurityPolicies.None; - SecureChannelContext verifiedContext = secure ? channelContext : new SecureChannelContext( - channelContext.SecureChannelId, channelContext.EndpointDescription, channelContext.MessageEncoding, - peerAddress: channelContext.PeerAddress) - { UpstreamIdentity = channelContext.UpstreamIdentity }; - if (m_classifier?.TryClassify(verifiedContext, current, out ResourceIsolationIdentity identity) == true || - m_classifier?.TryClassifyIngress( - channelContext.PeerAddress == null ? null : new IPEndPoint(channelContext.PeerAddress, 0), - out identity) == true) - { - string key = identity.Class == ResourceIsolationClass.Trusted - ? GetTrustedOwnerKey(identity.Key) : "mapped:" + identity.Key; - return identity.Class == owner.Class && key == owner.Key - ? StatusCodes.Good : StatusCodes.BadServerTooBusy; + if (m_classifier != null) + { + bool secure = original.Channel.SecurityMode is + MessageSecurityMode.Sign or MessageSecurityMode.SignAndEncrypt && + original.Channel.SecurityPolicyUri != SecurityPolicies.None; + SecureChannelContext verifiedContext = secure ? channelContext : new SecureChannelContext( + channelContext.SecureChannelId, channelContext.EndpointDescription, channelContext.MessageEncoding, + peerAddress: channelContext.PeerAddress) + { UpstreamIdentity = channelContext.UpstreamIdentity }; + if (m_classifier.TryClassify(verifiedContext, current, out ResourceIsolationIdentity identity) || + m_classifier.TryClassifyIngress( + channelContext.PeerAddress == null ? null : new IPEndPoint(channelContext.PeerAddress, 0), + out identity)) + { + string key = identity.Class == ResourceIsolationClass.Trusted + ? GetTrustedOwnerKey(identity.Key) : "mapped:" + identity.Key; + return identity.Class == owner.Class && key == owner.Key + ? StatusCodes.Good : StatusCodes.BadServerTooBusy; + } } ResourceIsolationClass expected = current != null && controlRequest ? ResourceIsolationClass.Control @@ -861,16 +864,35 @@ public ChannelSnapshot(SecureChannelContext context) /// public bool Matches(SecureChannelContext context) { - byte[]? peer = context.PeerAddress?.GetAddressBytes(); return m_channelId == context.SecureChannelId && SecurityPolicyUri == context.EndpointDescription?.SecurityPolicyUri && SecurityMode == (context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid) && (SecurityPolicyUri == SecurityPolicies.None || m_certificate.AsSpan().SequenceEqual(context.ClientChannelCertificate.AsSpan())) && - m_peer.AsSpan().SequenceEqual(peer.AsSpan()) && + PeerMatches(context.PeerAddress) && ReferenceEquals(m_upstreamIdentity, context.UpstreamIdentity); } + /// + /// Compares the captured peer address bytes; runs for every request, so the + /// current address is written to the stack rather than copied to a new array. + /// + private bool PeerMatches(IPAddress? peer) + { + if (peer == null) + { + return m_peer == null || m_peer.Length == 0; + } +#if NET + Span bytes = stackalloc byte[16]; + if (peer.TryWriteBytes(bytes, out int written)) + { + return m_peer.AsSpan().SequenceEqual(bytes[..written]); + } +#endif + return m_peer.AsSpan().SequenceEqual(peer.GetAddressBytes()); + } + private readonly string m_channelId; private readonly byte[]? m_certificate; private readonly byte[]? m_peer; diff --git a/src/Opc.Ua.Server/Server/OperationContext.cs b/src/Opc.Ua.Server/Server/OperationContext.cs index 1b1a44e046..d9bf203a62 100644 --- a/src/Opc.Ua.Server/Server/OperationContext.cs +++ b/src/Opc.Ua.Server/Server/OperationContext.cs @@ -395,8 +395,48 @@ internal void AttachSessionlessLease(IDisposable lease) m_sessionlessLease = lease; } + /// + /// Returns a copy of a node manager's system context bound to this operation, + /// reusing the copy made for the previous call with the same template. + /// + /// + /// Per-node node manager callbacks (permission and node metadata, reference + /// descriptions) otherwise clone the system context once for every node of a + /// request. The copies are equal for one request, so the last one is kept. + /// + /// The node manager's system context. + internal ServerSystemContext GetSystemContext(ServerSystemContext template) + { + SystemContextCacheEntry? entry = m_systemContextCache; + if (entry != null && ReferenceEquals(entry.Template, template)) + { + return entry.Copy; + } + + ServerSystemContext copy = template.Copy(this); + m_systemContextCache = new SystemContextCacheEntry(template, copy); + return copy; + } + + /// + /// A system context copy and the template it was made from, published as one + /// reference so concurrent readers never pair a copy with the wrong template. + /// + private sealed class SystemContextCacheEntry + { + public SystemContextCacheEntry(ServerSystemContext template, ServerSystemContext copy) + { + Template = template; + Copy = copy; + } + + public ServerSystemContext Template { get; } + public ServerSystemContext Copy { get; } + } + private IDisposable? m_requestScope; private IDisposable? m_sessionlessLease; + private SystemContextCacheEntry? m_systemContextCache; private static uint s_lastRequestId; } } diff --git a/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs b/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs index 4b9e0b1a6a..0d3264fb72 100644 --- a/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs +++ b/src/Opc.Ua.Server/Subscription/MonitoredItem/MonitoredItem.cs @@ -2341,7 +2341,6 @@ public static bool ValueChanged( } // select default data change filters. - const double deadband = 0.0; DeadbandType deadbandType = DeadbandType.None; DataChangeTrigger trigger = DataChangeTrigger.StatusValue; @@ -2361,11 +2360,8 @@ public static bool ValueChanged( } else { - filter = new DataChangeFilter - { - DeadbandType = (uint)deadbandType, - DeadbandValue = deadband - }; + // runs for every sampled value of an unfiltered item; AreEqual only reads it. + filter = s_defaultDataChangeFilter; } // get the current status. @@ -2713,6 +2709,16 @@ private void DisposeQueueHandlers() } private readonly Lock m_lock = new(); + + /// + /// The filter applied when a monitored item has none: no deadband, StatusValue trigger. + /// Shared and never modified. + /// + private static readonly DataChangeFilter s_defaultDataChangeFilter = new() + { + DeadbandType = (uint)DeadbandType.None, + DeadbandValue = 0.0 + }; private readonly ILogger m_logger; private bool m_isDisposed; private bool m_isAttaching; From e4867fcb5aa41e8d326eb8464246a45edc37e560 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 07:06:34 +0200 Subject: [PATCH 02/19] Add server load harness and trace analyzer under perf/ ServerLoadHarness runs a reference server in a child process and drives read/write/browse/subscription load against it, reporting client latency percentiles and the server's CPU, allocation, GC and lock-contention deltas, optionally recording the window with dotnet-trace. TraceAnalyzer aggregates those traces; ab.sh/ab-summary.ps1 run paired comparisons of two builds. Not part of UA.slnx. Co-Authored-By: Claude Opus 5.5 --- Directory.Packages.props | 1 + perf/.gitignore | 4 + perf/README.md | 76 ++ perf/ServerLoadHarness/Program.cs | 647 ++++++++++++++++++ .../ServerLoadHarness.csproj | 20 + perf/TraceAnalyzer/Program.cs | 174 +++++ perf/TraceAnalyzer/TraceAnalyzer.csproj | 17 + perf/ab-summary.ps1 | 25 + perf/ab.sh | 22 + 9 files changed, 986 insertions(+) create mode 100644 perf/.gitignore create mode 100644 perf/README.md create mode 100644 perf/ServerLoadHarness/Program.cs create mode 100644 perf/ServerLoadHarness/ServerLoadHarness.csproj create mode 100644 perf/TraceAnalyzer/Program.cs create mode 100644 perf/TraceAnalyzer/TraceAnalyzer.csproj create mode 100644 perf/ab-summary.ps1 create mode 100644 perf/ab.sh diff --git a/Directory.Packages.props b/Directory.Packages.props index 33df6f16f1..29a206953a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,6 +14,7 @@ + diff --git a/perf/.gitignore b/perf/.gitignore new file mode 100644 index 0000000000..ad11d0c47d --- /dev/null +++ b/perf/.gitignore @@ -0,0 +1,4 @@ +traces/ +*.nettrace +*.etlx +ab-results.txt diff --git a/perf/README.md b/perf/README.md new file mode 100644 index 0000000000..5c1a70ae36 --- /dev/null +++ b/perf/README.md @@ -0,0 +1,76 @@ +# Server performance tooling + +Tools for measuring server latency, throughput and GC cost under load. They are not part of +the product build (not in `UA.slnx`) and target `net10.0` only. + +| Path | Purpose | +|---|---| +| `ServerLoadHarness/` | Starts a reference server in a child process and drives load against it from the parent process. | +| `TraceAnalyzer/` | Aggregates the `dotnet-trace` files the harness records (CPU, allocations, lock contention). | +| `ab.sh`, `ab-summary.ps1` | Paired A/B runs of two harness builds, and the median summary. | + +The server runs in its own process, so the server-side numbers (CPU, allocated bytes, GC count +and pause time, `Monitor.LockContentionCount`) exclude the load generator. The NUnit +`LoadTest` fixture in `tests/Opc.Ua.Sessions.Tests` checks correctness under load; this harness +measures it. + +## Build + +```bash +CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release +dotnet build perf/TraceAnalyzer -c Release +dotnet tool install -g dotnet-trace +``` + +## Run + +```bash +perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe run --scenario read --nodes 100 --sessions 20 --duration 15 --warmup 8 +``` + +| Option | Default | Meaning | +|---|---|---| +| `--scenario` | `read` | `read`, `write`, `browse` (Server object) or `sub` (data-change subscriptions plus a writer). | +| `--sessions` | 20 | Concurrent sessions. | +| `--nodes` | 100 | Values per Read/Write request, or monitored items per subscription. | +| `--inflight` | 1 | Outstanding requests per session (closed loop). | +| `--security` | `none` | `none`, `sign` or `encrypt` (Basic256Sha256). | +| `--subs`, `--pub`, `--write-interval` | 5, 100, 100 | Subscriptions per session, publishing interval and writer interval in ms (`sub`). | +| `--duration`, `--warmup` | 20, 10 | Measurement window and warm-up in seconds. | +| `--diag`, `--audit` | 0 | Server diagnostics and auditing. | +| `--trace ` | | Record the measurement window with `dotnet-trace`. | +| `--trace-profile` | `cpu` | `cpu` (sampled thread time), `alloc` (allocation ticks), `contention` or `gc`. | +| `--out ` | | Append the result block to a file. | + +Environment variables prefixed `SERVER_` are passed to the server process without the prefix, +for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes. + +The result block reports client throughput and latency percentiles, the server deltas for the +window, and the server's allocated bytes and CPU time per request (or per notification). +Allocated bytes per operation are deterministic and the most reliable signal. On a shared +machine, throughput and latency vary from run to run, so compare builds with `ab.sh` rather +than with two separate runs. + +## Analyse a trace + +```bash +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll alloc trace.nettrace 40 +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll cpu trace.nettrace 40 Monitor.Enter_Slowpath +dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll contention trace.nettrace 15 +``` + +The sampled thread-time profile also samples blocked threads. The `cpu` mode drops samples +whose leaf frame is a known wait (thread-pool semaphore, I/O completion poll, GC suspension), +so the remaining samples approximate on-CPU time. Spin-waiting in `Monitor.Enter_Slowpath` or +`Lock.TryEnterSlow` is real CPU and shows up there. + +## A/B comparison + +Build the baseline harness in a second worktree, then alternate the two builds: + +```bash +git worktree add --detach C:/uabase origin/master +cp -r perf C:/uabase/ && (cd C:/uabase && CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release) +cd perf && BASELINE=C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe ./ab.sh +pwsh -File ab-summary.ps1 +``` diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs new file mode 100644 index 0000000000..a1bc5d8c73 --- /dev/null +++ b/perf/ServerLoadHarness/Program.cs @@ -0,0 +1,647 @@ +// Server load / profiling harness. Not part of the product; see perf/README.md. +// Modes: +// server --port P [--diag 0|1] [--audit 0|1] [--threads N] +// Starts a ReferenceServer and reads commands from stdin: mark | report | quit +// run --scenario read|write|sub|browse [--sessions N] [--nodes K] [--inflight I] +// [--duration S] [--warmup S] [--security none|sign|encrypt] [--diag 0|1] [--audit 0|1] +// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] +// Spawns the server child, drives the load and prints a report of client latency +// and server-side CPU / GC / allocation deltas for the measurement window. + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using Microsoft.Extensions.Logging; +using Opc.Ua; +using Opc.Ua.Client; +using Opc.Ua.Client.TestFramework; +using Opc.Ua.Configuration; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; + +internal static class Program +{ + private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); + + private static string Arg(string name, string def) => s_args.TryGetValue(name, out string? v) ? v : def; + private static int ArgInt(string name, int def) => int.Parse(Arg(name, def.ToString(CultureInfo.InvariantCulture)), CultureInfo.InvariantCulture); + + public static async Task Main(string[] args) + { + for (int i = 1; i < args.Length; i++) + { + if (args[i].StartsWith("--", StringComparison.Ordinal)) + { + string key = args[i][2..]; + string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) ? args[++i] : "1"; + s_args[key] = value; + } + } + string mode = args.Length > 0 ? args[0] : "run"; + return mode switch + { + "server" => await RunServerAsync().ConfigureAwait(false), + _ => await RunDriverAsync(args).ConfigureAwait(false) + }; + } + + // ------------------------------------------------------------------ server + + private static ITelemetryContext CreateTelemetry(LogLevel level) => + DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); + + private static async Task RunServerAsync() + { + int port = ArgInt("port", 62555); + bool diag = ArgInt("diag", 0) != 0; + bool audit = ArgInt("audit", 0) != 0; + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + + string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); + var application = new ApplicationInstance(telemetry) + { + ApplicationName = "PerfReferenceServer", + ApplicationType = ApplicationType.Server + }; + string endpointUrl = $"opc.tcp://localhost:{port}/PerfReferenceServer"; + ArrayOf certs = ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( + "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", CertificateStoreType.Directory, pkiRoot); + ApplicationConfiguration config = await application + .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") + .SetMaxByteStringLength(4 * 1024 * 1024) + .SetMaxArrayLength(1024 * 1024) + .SetMaxMessageSize(16 * 1024 * 1024) + .AsServer([endpointUrl]) + .AddUnsecurePolicyNone() + .AddSignPolicies() + .AddSignAndEncryptPolicies() + .SetMaxChannelCount(2100) + .SetDiagnosticsEnabled(diag) + .SetAuditingEnabled(audit) + .SetShutdownDelay(0) + .AddSecurityConfiguration(certs, pkiRoot) + .SetAutoAcceptUntrustedCertificates(true) + .CreateAsync().ConfigureAwait(false); + config.ServerConfiguration!.MaxSessionCount = 2000; + config.ServerConfiguration.MaxSubscriptionCount = 20000; + config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; + config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); + config.ServerConfiguration.MinRequestThreadCount = 50; + + if (!await application.CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime).ConfigureAwait(false)) + { + Console.WriteLine("ERROR cert"); + return 1; + } + + var server = new ReferenceServer(telemetry); + server.TransportBindings = TestTransportBindings.WithAllSchemes(); + await application.StartAsync(server).ConfigureAwait(false); + Console.WriteLine($"READY {endpointUrl} {Environment.ProcessId}"); + + ServerStats mark = ServerStats.Capture(); + string? line; + while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) + { + switch (line.Trim()) + { + case "mark": + GC.Collect(); + GC.WaitForPendingFinalizers(); + mark = ServerStats.Capture(); + Console.WriteLine("MARKED"); + break; + case "report": + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + break; + case "quit": + await server.StopAsync().ConfigureAwait(false); + return 0; + } + } + return 0; + } + + private sealed record ServerStats(long Alloc, int G0, int G1, int G2, TimeSpan Pause, TimeSpan Cpu, long Contention, long Stamp, int Threads, long WorkingSet, long Heap) + { + public static ServerStats Capture() + { + using var p = Process.GetCurrentProcess(); + return new ServerStats( + GC.GetTotalAllocatedBytes(true), GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), + GC.GetTotalPauseDuration(), p.TotalProcessorTime, Monitor.LockContentionCount, Stopwatch.GetTimestamp(), + ThreadPool.ThreadCount, p.WorkingSet64, GC.GetGCMemoryInfo().HeapSizeBytes); + } + + public string Delta(ServerStats m) + { + double sec = Stopwatch.GetElapsedTime(m.Stamp, Stamp).TotalSeconds; + return string.Create(CultureInfo.InvariantCulture, + $"sec={sec:F2} allocMB={(Alloc - m.Alloc) / 1048576.0:F1} gen0={G0 - m.G0} gen1={G1 - m.G1} gen2={G2 - m.G2} pauseMs={(Pause - m.Pause).TotalMilliseconds:F0} cpuMs={(Cpu - m.Cpu).TotalMilliseconds:F0} cpuCores={(Cpu - m.Cpu).TotalSeconds / sec:F2} contention={Contention - m.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); + } + } + + // ------------------------------------------------------------------ driver + + private static async Task RunDriverAsync(string[] args) + { + string scenario = Arg("scenario", "read"); + int sessionsCount = ArgInt("sessions", 20); + int durationSec = ArgInt("duration", 20); + int warmupSec = ArgInt("warmup", 10); + int port = ArgInt("port", 62555 + Environment.ProcessId % 1000); + string security = Arg("security", "none"); + + string exe = Environment.ProcessPath!; + var psi = new ProcessStartInfo(exe, $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}") + { + RedirectStandardInput = true, + RedirectStandardOutput = true, + UseShellExecute = false + }; + foreach (System.Collections.DictionaryEntry kv in Environment.GetEnvironmentVariables()) + { + string k = (string)kv.Key; + if (k.StartsWith("SERVER_", StringComparison.Ordinal)) + { + psi.Environment[k[7..]] = (string?)kv.Value; + } + } + using Process server = Process.Start(psi)!; + var serverLines = new BlockingCollection(); + _ = Task.Run(() => + { + string? l; + while ((l = server.StandardOutput.ReadLine()) != null) + { + if (l.StartsWith("READY", StringComparison.Ordinal) || l.StartsWith("REPORT", StringComparison.Ordinal) || l.StartsWith("MARKED", StringComparison.Ordinal) || l.StartsWith("ERROR", StringComparison.Ordinal)) + { + serverLines.Add(l); + } + else if (s_args.ContainsKey("verbose")) + { + Console.WriteLine("[server] " + l); + } + } + serverLines.CompleteAdding(); + }); + string ready = serverLines.Take(); + if (!ready.StartsWith("READY", StringComparison.Ordinal)) + { + Console.WriteLine(ready); + return 1; + } + string url = ready.Split(' ')[1]; + Console.WriteLine($"server pid={server.Id} url={url}"); + + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; + await client.LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient").ConfigureAwait(false); + string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; + ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); + MessageSecurityMode wantedMode = security switch + { + "sign" => MessageSecurityMode.Sign, + "encrypt" => MessageSecurityMode.SignAndEncrypt, + _ => MessageSecurityMode.None + }; + var filtered = new List(); + foreach (EndpointDescription ep in endpoints) + { + if (ep.SecurityPolicyUri == policy && ep.SecurityMode == wantedMode) + { + filtered.Add(ep); + } + } + + var sessions = new ISession[sessionsCount]; + var connectSw = Stopwatch.StartNew(); + await Parallel.ForAsync(0, sessionsCount, new ParallelOptions { MaxDegreeOfParallelism = 32 }, async (i, ct) => + sessions[i] = await client.ConnectAsync(new Uri(url), policy, filtered).ConfigureAwait(false)).ConfigureAwait(false); + Console.WriteLine($"connected {sessionsCount} sessions ({security}) in {connectSw.ElapsedMilliseconds} ms"); + + Workload workload = scenario switch + { + "sub" => new SubscribeWorkload(sessions, ArgInt("nodes", 100), ArgInt("subs", 5), ArgInt("pub", 100), ArgInt("write-interval", 100)), + _ => new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)) + }; + await workload.SetupAsync().ConfigureAwait(false); + + using var stop = new CancellationTokenSource(); + Task run = workload.RunAsync(stop.Token); + await Task.Delay(TimeSpan.FromSeconds(warmupSec)).ConfigureAwait(false); + + Process? tracer = null; + string? trace = s_args.TryGetValue("trace", out string? t) ? t : null; + if (trace != null) + { + string profile = Arg("trace-profile", "cpu"); + string providers = profile switch + { + // allocation sampling (GCAllocationTick ~ every 100KB) + GC events + "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5 --clrevents gc --clreventlevel verbose", + "gc" => "--profile gc-verbose", + "contention" => "--providers Microsoft-Windows-DotNETRuntime:0x4000:4", + _ => "--profile dotnet-sampled-thread-time" + }; + tracer = Process.Start(new ProcessStartInfo("dotnet-trace", $"collect --process-id {server.Id} {providers} -o \"{trace}\" --duration 00:00:{durationSec:D2}") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }); + await Task.Delay(2000).ConfigureAwait(false); + } + + server.StandardInput.WriteLine("mark"); + Take(serverLines, "MARKED"); + workload.ResetStats(); + var sw = Stopwatch.StartNew(); + await Task.Delay(TimeSpan.FromSeconds(durationSec)).ConfigureAwait(false); + server.StandardInput.WriteLine("report"); + string report = Take(serverLines, "REPORT"); + double elapsed = sw.Elapsed.TotalSeconds; + string clientReport = workload.Report(elapsed); + stop.Cancel(); + try + { + await run.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + if (tracer != null) + { + await tracer.WaitForExitAsync().ConfigureAwait(false); + Console.WriteLine("trace: " + trace); + } + + var sb = new StringBuilder(); + sb.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionsCount} {string.Join(' ', args.Skip(1))}\n"); + sb.Append(" client: ").Append(clientReport).Append('\n'); + sb.Append(" server: ").Append(report[7..]).Append('\n'); + double ops = workload.Operations; + if (ops > 0) + { + ServerReport sr = ServerReport.Parse(report[7..]); + sb.Append(CultureInfo.InvariantCulture, $" per-op: allocB={sr.AllocMb * 1048576 / ops:F0} cpuUs={sr.CpuMs * 1000 / ops:F1} ({workload.OperationName})\n"); + } + Console.Write(sb.ToString()); + if (s_args.TryGetValue("out", out string? outFile)) + { + File.AppendAllText(outFile, sb.ToString()); + } + + await Parallel.ForEachAsync(sessions, async (s, ct) => + { + try + { + await s.CloseAsync(ct).ConfigureAwait(false); + } + catch + { + } + s.Dispose(); + }).ConfigureAwait(false); + server.StandardInput.WriteLine("quit"); + if (!server.WaitForExit(10000)) + { + server.Kill(); + } + return 0; + } + + private static string Take(BlockingCollection lines, string prefix) + { + while (true) + { + string l = lines.Take(); + if (l.StartsWith(prefix, StringComparison.Ordinal)) + { + return l; + } + } + } + + private sealed record ServerReport(double AllocMb, double CpuMs) + { + public static ServerReport Parse(string s) + { + var d = s.Split(' ').Select(p => p.Split('=')).Where(p => p.Length == 2).ToDictionary(p => p[0], p => p[1]); + return new ServerReport(double.Parse(d["allocMB"], CultureInfo.InvariantCulture), double.Parse(d["cpuMs"], CultureInfo.InvariantCulture)); + } + } + + internal static List GetNodes(ISession session, int count, Type? onlyType = null) + { + var result = new List(); + foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + { + if (onlyType != null && type != onlyType) + { + continue; + } + foreach (ExpandedNodeId e in ids) + { + if (result.Count == count) + { + return result; + } + result.Add(ExpandedNodeId.ToNodeId(e, session.NamespaceUris)); + } + } + return result; + } +} + +internal abstract class Workload +{ + public abstract Task SetupAsync(); + public abstract Task RunAsync(CancellationToken ct); + public abstract void ResetStats(); + public abstract string Report(double seconds); + public abstract double Operations { get; } + public abstract string OperationName { get; } + + protected static string Percentiles(List ticksList) + { + if (ticksList.Count == 0) + { + return "n=0"; + } + long[] a = [.. ticksList]; + Array.Sort(a); + double Ms(double q) => a[Math.Min(a.Length - 1, (int)(q * a.Length))] * 1000.0 / Stopwatch.Frequency; + return string.Create(CultureInfo.InvariantCulture, + $"n={a.Length} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); + } +} + +internal sealed class RequestWorkload : Workload +{ + private readonly string m_kind; + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_inflight; + private readonly ConcurrentBag> m_lat = new(); + private readonly ThreadLocal> m_local; + private volatile int m_epoch; + private long m_requests; + private long m_errors; + private ArrayOf m_read; + private ArrayOf m_write; + private ArrayOf m_browse; + + public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) + { + m_kind = kind; + m_sessions = sessions; + m_nodes = nodes; + m_inflight = inflight; + m_local = new ThreadLocal>(() => + { + var l = new List(1 << 16); + m_lat.Add(l); + return l; + }, false); + } + + public override double Operations => Interlocked.Read(ref m_requests); + public override string OperationName => "request"; + + public override Task SetupAsync() + { + List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); + m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); + m_write = nodes.Select(n => new WriteValue { NodeId = n, AttributeId = Attributes.Value, Value = new DataValue(new Variant(42)) }).ToArray(); + m_browse = new[] { new BrowseDescription { NodeId = Opc.Ua.ObjectIds.Server, BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All } }; + Console.WriteLine($"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session"); + return Task.CompletedTask; + } + + public override async Task RunAsync(CancellationToken ct) + { + var tasks = new List(); + foreach (ISession s in m_sessions) + { + for (int i = 0; i < m_inflight; i++) + { + tasks.Add(Task.Run(() => LoopAsync(s, ct), CancellationToken.None)); + } + } + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + private async Task LoopAsync(ISession s, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + long start = Stopwatch.GetTimestamp(); + int epoch = m_epoch; + try + { + switch (m_kind) + { + case "write": + await s.WriteAsync(null, m_write, ct).ConfigureAwait(false); + break; + case "browse": + await s.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); + break; + default: + await s.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); + break; + } + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + if (Interlocked.Increment(ref m_errors) < 5) + { + Console.WriteLine("error: " + ex.Message); + } + continue; + } + long el = Stopwatch.GetTimestamp() - start; + if (epoch == m_epoch) + { + List l = m_local.Value!; + lock (l) + { + l.Add(el); + } + Interlocked.Increment(ref m_requests); + } + } + } + + public override void ResetStats() + { + m_epoch++; + foreach (List l in m_lat) + { + lock (l) + { + l.Clear(); + } + } + Interlocked.Exchange(ref m_requests, 0); + Interlocked.Exchange(ref m_errors, 0); + } + + public override string Report(double seconds) + { + var all = new List(); + foreach (List l in m_lat) + { + lock (l) + { + all.AddRange(l); + } + } + long req = Interlocked.Read(ref m_requests); + return string.Create(CultureInfo.InvariantCulture, + $"req/s={req / seconds:F0} values/s={req * (m_kind == "browse" ? 1 : m_nodes) / seconds:F0} errors={m_errors} latency {Percentiles(all)}"); + } +} + +internal sealed class SubscribeWorkload : Workload +{ + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_subs; + private readonly int m_pub; + private readonly int m_writeInterval; + private readonly long[] m_writeTicks = new long[1 << 16]; + private readonly ConcurrentBag> m_lat = new(); + private readonly ThreadLocal> m_local; + private long m_notifications; + private long m_writes; + private int m_counter; + private ISession? m_writer; + private ArrayOf m_nodeIds; + + public SubscribeWorkload(ISession[] sessions, int nodes, int subs, int pub, int writeInterval) + { + m_sessions = sessions; + m_nodes = nodes; + m_subs = subs; + m_pub = pub; + m_writeInterval = writeInterval; + m_local = new ThreadLocal>(() => + { + var l = new List(1 << 16); + m_lat.Add(l); + return l; + }, false); + } + + public override double Operations => Interlocked.Read(ref m_notifications); + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); + m_writer = m_sessions[0]; + var sw = Stopwatch.StartNew(); + await Parallel.ForEachAsync(m_sessions, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (session, ct) => + { + for (int j = 0; j < m_subs; j++) + { + var subscription = new Subscription(session.DefaultSubscription) + { + PublishingInterval = m_pub, + KeepAliveCount = 10, + LifetimeCount = 100 + }; + foreach (NodeId n in m_nodeIds) + { + subscription.AddItem(new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = n, + AttributeId = Attributes.Value, + MonitoringMode = MonitoringMode.Reporting, + SamplingInterval = 0, + QueueSize = 1 + }); + } + subscription.FastDataChangeCallback = (sub, notification, _) => + { + long now = Stopwatch.GetTimestamp(); + List l = m_local.Value!; + int count = 0; + lock (l) + { + foreach (MonitoredItemNotification item in notification.MonitoredItems) + { + if (item.Value.WrappedValue.TryGetValue(out int v) && v > 0) + { + long w = Volatile.Read(ref m_writeTicks[v & 0xFFFF]); + if (w != 0) + { + l.Add(now - w); + } + } + count++; + } + } + Interlocked.Add(ref m_notifications, count); + }; + session.AddSubscription(subscription); + await subscription.CreateAsync(ct).ConfigureAwait(false); + } + }).ConfigureAwait(false); + Console.WriteLine($"sub: {m_sessions.Length} sessions x {m_subs} subs x {m_nodeIds.Count} items = {m_sessions.Length * m_subs * m_nodeIds.Count} items, pub {m_pub} ms, write every {m_writeInterval} ms; setup {sw.ElapsedMilliseconds} ms"); + } + + public override async Task RunAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); + while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) + { + int v = Interlocked.Increment(ref m_counter); + var values = new WriteValue[m_nodeIds.Count]; + for (int i = 0; i < values.Length; i++) + { + values[i] = new WriteValue { NodeId = m_nodeIds[i], AttributeId = Attributes.Value, Value = new DataValue(new Variant(v)) }; + } + Volatile.Write(ref m_writeTicks[v & 0xFFFF], Stopwatch.GetTimestamp()); + await m_writer!.WriteAsync(null, values, ct).ConfigureAwait(false); + Interlocked.Increment(ref m_writes); + } + } + + public override void ResetStats() + { + foreach (List l in m_lat) + { + lock (l) + { + l.Clear(); + } + } + Interlocked.Exchange(ref m_notifications, 0); + Interlocked.Exchange(ref m_writes, 0); + } + + public override string Report(double seconds) + { + var all = new List(); + foreach (List l in m_lat) + { + lock (l) + { + all.AddRange(l); + } + } + long n = Interlocked.Read(ref m_notifications); + long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subs * m_nodeIds.Count; + return string.Create(CultureInfo.InvariantCulture, + $"notif/s={n / seconds:F0} received/expected={(expected == 0 ? 0 : (double)n / expected):P1} write->notify latency {Percentiles(all)}"); + } +} diff --git a/perf/ServerLoadHarness/ServerLoadHarness.csproj b/perf/ServerLoadHarness/ServerLoadHarness.csproj new file mode 100644 index 0000000000..1d238e7b18 --- /dev/null +++ b/perf/ServerLoadHarness/ServerLoadHarness.csproj @@ -0,0 +1,20 @@ + + + Exe + net10.0 + enable + enable + false + false + true + true + $(NoWarn);CS1591;CA1303;CA2007;CA1031;CA2000;CA1849;CA5394;RCS1090;CA1515;CA1305;CA1307;CA1310;IDE0005;CA1062;CA1822;CA1859;CA1861;CS0618;RCS1163;CA1002;CA1812;CA1852;CA1050;CA2201;CA1001 + false + false + none + + + + + + diff --git a/perf/TraceAnalyzer/Program.cs b/perf/TraceAnalyzer/Program.cs new file mode 100644 index 0000000000..a8002ec6eb --- /dev/null +++ b/perf/TraceAnalyzer/Program.cs @@ -0,0 +1,174 @@ +// Trace analyzer for perf/ServerLoadHarness traces. Not part of the product; see perf/README.md. +// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped), exclusive/inclusive, callers of focus +// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, by first Opc.Ua frame, and stacks +// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack +using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tracing.Etlx; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +string mode = args[0]; +string file = args[1]; +int top = args.Length > 2 ? int.Parse(args[2]) : 40; +string focus = args.Length > 3 ? args[3] : null; +string[] s_blocked = { "LowLevelLifoSemaphore.Wait", "WaitForSignal", "WorkerThreadStart", "IOCompletionPoller.Poll", "WaitOneNoCheck", "Thread.Sleep", "PollGCWorker|70_0", "WaitMultipleIgnoringSyncContext", "Monitor.Wait", "!?" }; + +string etlx = TraceLog.CreateFromEventPipeDataFile(file, null, new TraceLogOptions { ContinueOnError = true }); +using var log = new TraceLog(etlx); + +static string Name(TraceCallStack s) +{ + var m = s.CodeAddress.FullMethodName; + if (string.IsNullOrEmpty(m)) + { + m = s.CodeAddress.ModuleName + "!?"; + } + return m; +} + +static string Short(string m) +{ + int p = m.IndexOf('('); + return p > 0 ? m[..p] : m; +} + +static bool IsOurs(string m) => m.StartsWith("Opc.Ua", StringComparison.Ordinal) || m.StartsWith("Quickstarts", StringComparison.Ordinal); + +if (mode == "contention") +{ + var byStack = new Dictionary(); + int total = 0; + foreach (TraceEvent e in log.Events) + { + if (e.EventName == null || !e.EventName.StartsWith("Contention/Start")) continue; + total++; + var chain = new List(); + for (TraceCallStack f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) { string n = Short(Name(f)); if (IsOurs(n) || chain.Count == 0) chain.Add(n); } + string k = string.Join("\n ", chain); + byStack[k] = byStack.GetValueOrDefault(k) + 1; + } + Console.WriteLine($"contention events: {total}"); + foreach (var kv in byStack.OrderByDescending(k => k.Value).Take(top)) Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + return; +} +if (mode == "cpu") +{ + var excl = new Dictionary(); + var incl = new Dictionary(); + var firstOurs = new Dictionary(); + var callers = new Dictionary(); + int total = 0, all = 0; + foreach (TraceEvent e in log.Events) + { + if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") + { + continue; + } + all++; + object type = e.PayloadByName("Type"); + if (type != null && Convert.ToInt32(type) != 2) + { + continue; // not running managed code + } + TraceCallStack s = e.CallStack(); + if (s == null) + { + continue; + } + string leaf = Short(Name(s)); + if (s_blocked.Any(b => leaf.EndsWith(b))) continue; + total++; + excl[leaf] = excl.GetValueOrDefault(leaf) + 1; + var seen = new HashSet(); + bool gotOurs = false; + bool focusHit = false; + for (TraceCallStack f = s; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (seen.Add(n)) + { + incl[n] = incl.GetValueOrDefault(n) + 1; + } + if (!gotOurs && IsOurs(n)) + { + gotOurs = true; + firstOurs[n] = firstOurs.GetValueOrDefault(n) + 1; + } + if (focus != null && !focusHit && n.Contains(focus)) + { + focusHit = true; + string c = f.Caller != null ? Short(Name(f.Caller)) : ""; + callers[c] = callers.GetValueOrDefault(c) + 1; + } + } + } + Console.WriteLine($"samples: managed-running={total} all={all}"); + void Dump(string title, Dictionary d, int n) + { + Console.WriteLine($"--- {title}"); + foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + { + Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + } + } + Dump("exclusive", excl, top); + Dump("first Opc.Ua frame (exclusive-ish)", firstOurs, top); + Dump("inclusive (Opc.Ua only)", incl.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value), top); + if (focus != null) + { + Dump("callers of " + focus, callers, top); + } +} +else +{ + var byType = new Dictionary(); + var byFrame = new Dictionary(); + var byStack = new Dictionary(); + double total = 0; + foreach (TraceEvent e in log.Events) + { + if (e is not GCAllocationTickTraceData a) + { + continue; + } + double amount = a.AllocationAmount64; + total += amount; + string t = a.TypeName ?? "?"; + byType[t] = byType.GetValueOrDefault(t) + amount; + TraceCallStack s = e.CallStack(); + string frame = "?"; + var chain = new List(); + for (TraceCallStack f = s; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (IsOurs(n)) + { + if (frame == "?") + { + frame = n; + } + if (chain.Count < 6) + { + chain.Add(n); + } + } + } + byFrame[t + " <- " + frame] = byFrame.GetValueOrDefault(t + " <- " + frame) + amount; + if (focus == null || t.Contains(focus)) + { + string k = t + "\n " + string.Join("\n ", chain); + byStack[k] = byStack.GetValueOrDefault(k) + amount; + } + } + Console.WriteLine($"sampled alloc total: {total / 1048576:F1} MB"); + void Dump(string title, Dictionary d, int n) + { + Console.WriteLine($"--- {title}"); + foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + { + Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + } + } + Dump("by type", byType, top); + Dump("by type <- first Opc.Ua frame", byFrame, top); + Dump("stacks", byStack, top / 2); +} diff --git a/perf/TraceAnalyzer/TraceAnalyzer.csproj b/perf/TraceAnalyzer/TraceAnalyzer.csproj new file mode 100644 index 0000000000..e2529b9c27 --- /dev/null +++ b/perf/TraceAnalyzer/TraceAnalyzer.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + disable + enable + false + false + false + none + $(NoWarn);CS1591;CA1305;CA1307;CA1310;CA1849;RCS1007;IDE0011 + + + + + diff --git a/perf/ab-summary.ps1 b/perf/ab-summary.ps1 new file mode 100644 index 0000000000..6f2ca38803 --- /dev/null +++ b/perf/ab-summary.ps1 @@ -0,0 +1,25 @@ +param($file = "ab-results.txt") +$rows = @() +$cur = $null +foreach ($l in Get-Content $file) { + if ($l -match '^### side=(\w) round=(\d) (.*)$') { $cur = @{ side = $Matches[1]; scen = $Matches[3].Trim() }; continue } + if ($null -eq $cur) { continue } + if ($l -match 'req/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'notif/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'p50=([\d.]+)ms p90=([\d.]+)ms p99=([\d.]+)ms') { $cur.p50 = [double]$Matches[1]; $cur.p90 = [double]$Matches[2]; $cur.p99 = [double]$Matches[3] } + if ($l -match 'pauseMs=(\d+)') { $cur.pause = [double]$Matches[1] } + if ($l -match 'contention=(\d+)') { $cur.cont = [double]$Matches[1] } + if ($l -match 'per-op: allocB=(\d+) cpuUs=([\d.]+)') { $cur.alloc = [double]$Matches[1]; $cur.cpu = [double]$Matches[2]; $rows += [pscustomobject]$cur; $cur = $null } +} +function Med($xs) { $s = @($xs | Sort-Object); if ($s.Count -eq 0) { return [double]::NaN }; return $s[[int][Math]::Floor($s.Count / 2)] } +$out = @() +foreach ($g in ($rows | Group-Object scen)) { + $a = $g.Group | Where-Object side -eq 'A'; $b = $g.Group | Where-Object side -eq 'B' + $o = [ordered]@{ scenario = $g.Name; n = "$(@($a).Count)/$(@($b).Count)" } + foreach ($m in 'tput', 'p50', 'p90', 'p99', 'alloc', 'cpu', 'pause', 'cont') { + $ma = Med ($a | ForEach-Object { $_.$m }); $mb = Med ($b | ForEach-Object { $_.$m }) + $o["$m"] = "{0:G4} -> {1:G4} ({2:+0;-0}%)" -f $ma, $mb, (100 * ($mb - $ma) / $ma) + } + $out += [pscustomobject]$o +} +$out | Format-List diff --git a/perf/ab.sh b/perf/ab.sh new file mode 100644 index 0000000000..3f26ae6dcd --- /dev/null +++ b/perf/ab.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# BASELINE: harness built in a second worktree of the commit to compare against. +A=${BASELINE:-C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe} +B=./ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe +O=ab-results.txt; rm -f $O +while IFS= read -r s; do + for r in 1 2 3; do + for side in A B; do + exe=$A; [ $side = B ] && exe=$B + echo "### side=$side round=$r $s" >> $O + timeout 300 $exe run $s --sessions 20 --duration 12 --warmup 6 --out $O >/dev/null 2>&1 + done + done +done <<'S' +--scenario read --nodes 1 +--scenario read --nodes 100 +--scenario write --nodes 100 +--scenario browse +--scenario read --nodes 100 --security encrypt +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 +S +echo done >> $O From 9068684c39aefee61af99b781f99ab527dc1a97e Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 07:19:00 +0200 Subject: [PATCH 03/19] Server: validate node permissions without allocating NodeMetadata Read, Write, Browse and Call validate every node they touch, and the dispatcher discarded the NodeMetadata (248 B, plus copies of the role permission arrays) right after the check. Browse also read 13 attributes per reference target only to check its RolePermissions. The role-permission and access-restriction validators now run on an internal PermissionMetadata struct holding exactly the attributes they read; the NodeMetadata overloads build it and delegate, so the checks are unchanged. AsyncCustomNodeManager fills the struct on the stack from the same three attributes and namespace defaults that GetPermissionMetadataAsync reads. The dispatcher uses it when the metadata is not needed, and Browse's reference filter uses it instead of GetNodeMetadataAsync. Unresolved handles still take the NodeMetadata path with its fallbacks. Read 100 values: alloc/request 120.6 -> 95.7 KB, ~21k -> 26k req/s; Browse: 67.6 -> 51.8 KB, 26.7k -> 29.8k req/s. Co-Authored-By: Claude Opus 5.5 --- .../NodeManager/AsyncCustomNodeManager.cs | 154 +++++++++++++++++- .../MasterNodeManager.ServiceDispatch.cs | 40 ++++- .../NodeManagerServiceDispatcher.cs | 27 +++ .../NodeManager/PermissionMetadata.cs | 104 ++++++++++++ 4 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs diff --git a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs index 88cb9791b8..34c27617f1 100644 --- a/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs +++ b/src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs @@ -9204,6 +9204,25 @@ public virtual async ValueTask ValidateRolePermissionsAsync( return StatusCodes.Good; } + // Browse validates every reference target: read only the permission + // attributes instead of the full node metadata. + if (nodeManager is AsyncCustomNodeManager customNodeManager) + { + (bool handled, ServiceResult? verdict) = await customNodeManager.TryValidatePermissionsAsync( + operationContext, + nodeHandle, + requestedPermission, + null, + permissionsOnly: true, + logger: null, + validateAccessRestrictions: false, + cancellationToken).ConfigureAwait(false); + if (handled) + { + return verdict!; + } + } + NodeMetadata nodeMetadata = await nodeManager.GetNodeMetadataAsync( operationContext, nodeHandle, @@ -10460,6 +10479,133 @@ private async ValueTask SetDefaultPermissionsAsync( NodeMetadata metadata, CancellationToken cancellationToken = default) { + (AccessRestrictionType defaultAccessRestrictions, + ArrayOf defaultRolePermissions, + ArrayOf defaultUserRolePermissions) = + await GetDefaultPermissionsAsync(systemContext, target, cancellationToken).ConfigureAwait(false); + metadata.DefaultAccessRestrictions = defaultAccessRestrictions; + metadata.DefaultRolePermissions = defaultRolePermissions; + metadata.DefaultUserRolePermissions = defaultUserRolePermissions; + } + + /// + /// Validates the role permissions and access restrictions of a node for the requested + /// permission, reading the same attributes as + /// without materializing a . + /// + /// + /// Handled is false when the handle does not resolve to a node of this + /// node manager; the caller then takes the path, which has + /// the fallbacks for that case. + /// + internal async ValueTask<(bool Handled, ServiceResult? Result)> TryValidatePermissionsAsync( + OperationContext context, + object targetHandle, + PermissionType requestedPermission, + Dictionary? uniqueNodesServiceAttributesCache, + bool permissionsOnly, + ILogger? logger, + bool validateAccessRestrictions, + CancellationToken cancellationToken = default) + { + ServerSystemContext systemContext = GetOperationSystemContext(context); + + NodeHandle? handle = IsHandleInNamespace(targetHandle); + if (handle == null) + { + return (false, null); + } + + NodeState? target = await ValidateNodeAsync(systemContext, handle, null!, cancellationToken).ConfigureAwait(false); + if (target == null) + { + return (false, null); + } + + Variant[]? values = null; + if (uniqueNodesServiceAttributesCache != null) + { + NodeId key = handle.NodeId; + if (!uniqueNodesServiceAttributesCache.TryGetValue(key, out values) || + values.Length != 3) + { + values = new Variant[3]; + ReadAndCacheValidationAttributes( + uniqueNodesServiceAttributesCache, + systemContext, + target, + key, + ref values); + } + } + else if (permissionsOnly) + { + values = new Variant[3]; + ReadValidationAttributes(systemContext, target, ref values); + } + + // the same conversions as SetAccessAndRolePermissions; the arrays are only read + // for the verdict, so they are not copied. + AccessRestrictionType accessRestrictions = AccessRestrictionType.None; + ArrayOf rolePermissions = default; + ArrayOf userRolePermissions = default; + if (values != null) + { + if (values[0].TryGetValue(out ushort restrictions)) + { + accessRestrictions = (AccessRestrictionType)restrictions; + } + if (values[1].TryGetStructure(out ArrayOf roles)) + { + rolePermissions = roles; + } + if (values[2].TryGetStructure(out ArrayOf userRoles)) + { + userRolePermissions = userRoles; + } + } + + (AccessRestrictionType defaultAccessRestrictions, + ArrayOf defaultRolePermissions, + ArrayOf defaultUserRolePermissions) = + await GetDefaultPermissionsAsync(systemContext, target, cancellationToken).ConfigureAwait(false); + + var permissions = new PermissionMetadata( + target.NodeId, + target.IsPartOfTypeHierarchy, + accessRestrictions, + defaultAccessRestrictions, + rolePermissions, + defaultRolePermissions, + userRolePermissions, + defaultUserRolePermissions); + + ServiceResult result = MasterNodeManager.ValidateRolePermissions( + context, + permissions, + requestedPermission, + logger); + return (true, validateAccessRestrictions && ServiceResult.IsGood(result) + ? MasterNodeManager.ValidateAccessRestrictions(context, permissions, requestedPermission) + : result); + } + + /// + /// Reads the namespace default values for DefaultAccessRestrictions, DefaultRolePermissions + /// and DefaultUserRolePermissions of the node's namespace. + /// + private async ValueTask<( + AccessRestrictionType DefaultAccessRestrictions, + ArrayOf DefaultRolePermissions, + ArrayOf DefaultUserRolePermissions)> GetDefaultPermissionsAsync( + ServerSystemContext systemContext, + NodeState target, + CancellationToken cancellationToken = default) + { + AccessRestrictionType defaultAccessRestrictions = AccessRestrictionType.None; + ArrayOf defaultRolePermissions = default; + ArrayOf defaultUserRolePermissions = default; + // check if NamespaceMetadata is defined for NamespaceIndex of the node. NamespaceMetadataState? namespaceMetadataState = await Server.NodeManager.ConfigurationNodeManager!.GetNamespaceMetadataStateAsync(target.NodeId.NamespaceIndex, cancellationToken) @@ -10482,7 +10628,7 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull) { - metadata.DefaultAccessRestrictions = + defaultAccessRestrictions = value.GetEnumeration(); } } @@ -10499,7 +10645,7 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull && value.TryGetStructure(out ArrayOf rolePermissions)) { - metadata.DefaultRolePermissions = rolePermissions; + defaultRolePermissions = rolePermissions; } } @@ -10515,10 +10661,12 @@ private async ValueTask SetDefaultPermissionsAsync( if (!value.IsNull && value.TryGetStructure(out ArrayOf userRolePermissions)) { - metadata.DefaultUserRolePermissions = userRolePermissions; + defaultUserRolePermissions = userRolePermissions; } } } + + return (defaultAccessRestrictions, defaultRolePermissions, defaultUserRolePermissions); } /// diff --git a/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs b/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs index 7f5196bce6..5ec88af821 100644 --- a/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs +++ b/src/Opc.Ua.Server/NodeManager/MasterNodeManager.ServiceDispatch.cs @@ -874,7 +874,7 @@ protected ValueTask ValidatePermissionsAsync( /// hierarchy visibility exemption. /// private static bool IsTypeHierarchyVisibilityRequest( - NodeMetadata nodeMetadata, + in PermissionMetadata nodeMetadata, PermissionType requestedPermission) { return nodeMetadata.IsPartOfTypeHierarchy && @@ -910,6 +910,20 @@ protected internal static ServiceResult ValidateAccessRestrictions( OperationContext context, NodeMetadata nodeMetadata, PermissionType requestedPermission) + { + return ValidateAccessRestrictions( + context, + PermissionMetadata.From(nodeMetadata), + requestedPermission); + } + + /// + /// Validate the AccessRestrictions attribute for the requested permission. + /// + internal static ServiceResult ValidateAccessRestrictions( + OperationContext context, + in PermissionMetadata nodeMetadata, + PermissionType requestedPermission) { ServiceResult serviceResult = StatusCodes.Good; @@ -993,7 +1007,29 @@ protected internal static ServiceResult ValidateRolePermissions( PermissionType requestedPermission, ILogger? logger = null) { - if (nodeMetadata == null || requestedPermission == PermissionType.None) + if (nodeMetadata == null) + { + // no permission is required hence the validation passes + return StatusCodes.Good; + } + + return ValidateRolePermissions( + context, + PermissionMetadata.From(nodeMetadata), + requestedPermission, + logger); + } + + /// + /// Validates the role permissions + /// + internal static ServiceResult ValidateRolePermissions( + OperationContext context, + in PermissionMetadata nodeMetadata, + PermissionType requestedPermission, + ILogger? logger = null) + { + if (requestedPermission == PermissionType.None) { // no permission is required hence the validation passes return StatusCodes.Good; diff --git a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs index 251eeda8d1..a98a50f707 100644 --- a/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs +++ b/src/Opc.Ua.Server/NodeManager/NodeManagerServiceDispatcher.cs @@ -3851,6 +3851,33 @@ internal async ValueTask ValidatePermissionsAsync( return (StatusCodes.Good, null); } + // Most callers only need the verdict: validate on the stack instead of + // materializing NodeMetadata for every node of the request. + if (!metadataRequired && nodeManager is AsyncCustomNodeManager customNodeManager) + { + try + { + (bool handled, ServiceResult? verdict) = await customNodeManager.TryValidatePermissionsAsync( + context, + nodeHandle, + requestedPermission, + uniqueNodesServiceAttributes, + permissionsOnly, + m_logger, + validateAccessRestrictions: true, + cancellationToken) + .ConfigureAwait(false); + if (handled) + { + return (verdict!, null); + } + } + catch (ServiceResultException exception) + { + return (new ServiceResult(exception), null); + } + } + // First attempt to retrieve just the Permission metadata with or without cache optimization // If it happens that nodemanager does not fully implement GetPermissionMetadata, // fallback to GetNodeMetadataAsync diff --git a/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs b/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs new file mode 100644 index 0000000000..3d8cff7236 --- /dev/null +++ b/src/Opc.Ua.Server/NodeManager/PermissionMetadata.cs @@ -0,0 +1,104 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +namespace Opc.Ua.Server +{ + /// + /// The attributes the role-permission and access-restriction checks read: + /// the subset of a permission check needs. + /// + /// + /// Read, Write, Browse and Call validate every node they touch. Filling this + /// value on the stack spares the per-node object when + /// the caller only needs the verdict. + /// + internal readonly struct PermissionMetadata + { + public PermissionMetadata( + NodeId nodeId, + bool isPartOfTypeHierarchy, + AccessRestrictionType accessRestrictions, + AccessRestrictionType defaultAccessRestrictions, + ArrayOf rolePermissions, + ArrayOf defaultRolePermissions, + ArrayOf userRolePermissions, + ArrayOf defaultUserRolePermissions) + { + NodeId = nodeId; + IsPartOfTypeHierarchy = isPartOfTypeHierarchy; + AccessRestrictions = accessRestrictions; + DefaultAccessRestrictions = defaultAccessRestrictions; + RolePermissions = rolePermissions; + DefaultRolePermissions = defaultRolePermissions; + UserRolePermissions = userRolePermissions; + DefaultUserRolePermissions = defaultUserRolePermissions; + } + + /// + /// Takes the permission attributes of node metadata. + /// + public static PermissionMetadata From(NodeMetadata metadata) + { + return new PermissionMetadata( + metadata.NodeId, + metadata.IsPartOfTypeHierarchy, + metadata.AccessRestrictions, + metadata.DefaultAccessRestrictions, + metadata.RolePermissions, + metadata.DefaultRolePermissions, + metadata.UserRolePermissions, + metadata.DefaultUserRolePermissions); + } + + /// + public NodeId NodeId { get; } + + /// + public bool IsPartOfTypeHierarchy { get; } + + /// + public AccessRestrictionType AccessRestrictions { get; } + + /// + public AccessRestrictionType DefaultAccessRestrictions { get; } + + /// + public ArrayOf RolePermissions { get; } + + /// + public ArrayOf DefaultRolePermissions { get; } + + /// + public ArrayOf UserRolePermissions { get; } + + /// + public ArrayOf DefaultUserRolePermissions { get; } + } +} From 936b28d76059ae669a87b52b87b02a64ed093b31 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:08 +0200 Subject: [PATCH 04/19] Server: expose channel certificates as read-only ByteString; halve session lookup locking SecureChannelContext.ClientChannelCertificate, ServerChannelCertificate and ChannelThumbprint are ByteString instead of byte[]. The TCP listener now shares one cached DER copy per channel with every request, and a mutable array handed to request handlers or IResourceIsolationClassifier code could have altered the evidence later requests are checked against. A ByteString is a read-only view, so the buffer is shared safely without copying. The constructor keeps its byte[] parameters and wraps them; a missing value is IsNull, so the session signature data (which skips a null certificate but hashes an empty one) is unchanged. Readers in Server, GDS, LDS and the Redundancy session mirror are migrated; the mirrored activation state keeps recording an empty, not null, certificate. MigrationGuide.md lists the mapping for application code. SessionManager.TryGetSessionContext runs at admission and at dispatch of every request and took the global bindings lock twice per call. The candidate snapshot now reads the concurrent session index and the activation-state table without the lock; the validating check under the lock still confirms the binding is the committed one. Co-Authored-By: Claude Opus 5.5 --- docs/MigrationGuide.md | 26 ++++++ .../Stack/Server/SecureChannelContext.cs | 31 +++++-- .../Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs | 3 +- .../AuthorizationHelper.cs | 9 +- src/Opc.Ua.Lds.Server/LdsServer.cs | 6 +- .../Sessions/DistributedSessionManager.cs | 15 +++- .../DefaultServerResourceIsolationProvider.cs | 12 +-- .../Session/ChannelEvidenceExtensions.cs | 48 ++++++++++ src/Opc.Ua.Server/Session/Session.cs | 30 +++---- src/Opc.Ua.Server/Session/SessionManager.cs | 44 ++++----- .../Session/SessionSecurityPolicyHelper.cs | 6 +- .../Stack/Server/SecureChannelContextTests.cs | 89 +++++++++++++++++++ ...istributedSessionMirrorIntegrationTests.cs | 2 +- .../ServerFixtureUtils.cs | 6 +- .../RuntimeResourceIsolationTests.cs | 2 +- .../EphemeralNonceLifetimeRegressionTests.cs | 4 +- .../SessionRestoreTests.cs | 2 +- .../SessionSecurityTests.cs | 24 ++--- 18 files changed, 275 insertions(+), 84 deletions(-) create mode 100644 src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs create mode 100644 tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index b9b2302357..9ce6b5f1fa 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -33,6 +33,7 @@ covers cross-cutting changes. - [Transport resource limits](#transport-resource-limits) - [Migrating channel subclasses that override HandleIncomingMessage](#migrating-channel-subclasses-that-override-handleincomingmessage) - [Migrating custom IUserDatabase implementations](#migrating-custom-iuserdatabase-implementations) +- [Reading certificates from SecureChannelContext](#reading-certificates-from-securechannelcontext) - [Write service Value semantics follow OPC 10000-3 and OPC 10000-4](#write-service-value-semantics-follow-opc-10000-3-and-opc-10000-4) - [Address space permissions, Call, NodeManagement and locales follow OPC 10000-3](#address-space-permissions-call-nodemanagement-and-locales-follow-opc-10000-3) - [Migrating from 1.05.377 to 1.05.378](#migrating-from-105377-to-105378) @@ -908,6 +909,31 @@ and no longer keeps metadata only in memory, so a store that cannot persist metadata should reject the write by returning `false` rather than silently accepting it. +## Reading certificates from SecureChannelContext + +`SecureChannelContext.ClientChannelCertificate`, `ServerChannelCertificate` and +`ChannelThumbprint` are `ByteString` instead of `byte[]`. The transport now shares +one buffer of a channel's certificates with every request on that channel instead +of copying the DER encoding per request, and a `ByteString` is a read-only view of +it, so a request handler or an `IResourceIsolationClassifier` cannot alter the +evidence later requests are checked against. + +The constructor still takes `byte[]` and wraps it without copying. A missing +value is `IsNull`; an empty array stays empty, not null. + +| Before | After | +|---|---| +| `context.ClientChannelCertificate == null` | `context.ClientChannelCertificate.IsNull` | +| `context.ClientChannelCertificate?.Length > 0` | `context.ClientChannelCertificate.Length > 0` | +| `context.ClientChannelCertificate.AsSpan()` | `context.ClientChannelCertificate.Span` | +| `Certificate.FromRawData(context.ClientChannelCertificate)` | `Certificate.FromRawData(context.ClientChannelCertificate.Memory)` | +| `context.ClientChannelCertificate.ToByteString()` | `context.ClientChannelCertificate` | +| passing the array to an API that takes `byte[]?` | `value.IsNull ? null : value.ToArray()` | + +Keep a missing certificate null when converting: the session signature helpers +hash an empty array but skip a null one, so substituting `ToArray()` for a null +value changes the signature data. + ## ContentFilter NULL semantics follow OPC 10000-4 1.05.07 `FilterEvaluator` applies the NULL rules of diff --git a/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs b/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs index ec02e0ae56..62e62ba1c9 100644 --- a/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs +++ b/src/Opc.Ua.Core/Stack/Server/SecureChannelContext.cs @@ -58,9 +58,11 @@ public SecureChannelContext( SecureChannelId = secureChannelId; EndpointDescription = endpointDescription; MessageEncoding = messageEncoding; - ClientChannelCertificate = clientChannelCertificate; - ServerChannelCertificate = serverChannelCertificate; - ChannelThumbprint = channelThumbprint; + // Wrapped, not copied: the transport passes buffers it owns for the life of + // the channel, and ByteString hands out read-only views only. + ClientChannelCertificate = clientChannelCertificate == null ? default : new ByteString(clientChannelCertificate); + ServerChannelCertificate = serverChannelCertificate == null ? default : new ByteString(serverChannelCertificate); + ChannelThumbprint = channelThumbprint == null ? default : new ByteString(channelThumbprint); PeerAddress = peerAddress; } @@ -88,17 +90,30 @@ public SecureChannelContext( /// /// The unique hash for the secure channel calculated during channel creation. /// - public byte[]? ChannelThumbprint { get; } + /// + /// Read-only: the transport shares one buffer with every request of the channel. + /// when the channel has no thumbprint. + /// + public ByteString ChannelThumbprint { get; } /// - /// The client certificate used to establsih the secure channel. + /// The DER encoded client certificate used to establish the secure channel. /// - public byte[]? ClientChannelCertificate { get; } + /// + /// Read-only: the transport shares one buffer with every request of the channel, + /// so request handlers and resource-isolation classifiers cannot alter the evidence + /// later requests are checked against. when the + /// channel is not secured with a client certificate. + /// + public ByteString ClientChannelCertificate { get; } /// - /// The server certificate used to establsih the secure channel. + /// The DER encoded server certificate used to establish the secure channel. /// - public byte[]? ServerChannelCertificate { get; } + /// + /// Read-only, see . + /// + public ByteString ServerChannelCertificate { get; } /// /// The observed network address of the peer, when the transport exposes one. diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs index e0ff3022f8..b9ef57c6e2 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Asymmetric.cs @@ -104,7 +104,8 @@ protected string SecurityPolicyUri /// /// /// Certificate.RawData returns a new copy on every access; the listener needs the - /// bytes for every request it dispatches. + /// bytes for every request it dispatches. The array never leaves the stack: + /// exposes it as a read-only ByteString view. /// internal byte[]? ClientCertificateRawData => GetCachedRawData(ClientCertificate, ref m_clientCertificateRawData); diff --git a/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs b/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs index 0633acb167..fcd0627157 100644 --- a/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs +++ b/src/Opc.Ua.Gds.Server/RoleBasedUserManagement/AuthorizationHelper.cs @@ -298,10 +298,7 @@ public static void HasAuthenticatedSecureChannel( public static ByteString GetClientCertificateFingerprint(ISystemContext context) { OperationContext operationContext = GetOperationContext(context); - byte[] clientCertificate = operationContext.ChannelContext?.ClientChannelCertificate ?? - throw new ServiceResultException( - StatusCodes.BadSecurityChecksFailed, - "Unable to bind the request to a SecureChannel client certificate."); + ByteString clientCertificate = operationContext.ChannelContext?.ClientChannelCertificate ?? default; if (clientCertificate.Length == 0) { @@ -311,10 +308,10 @@ public static ByteString GetClientCertificateFingerprint(ISystemContext context) } #if NET6_0_OR_GREATER - return ByteString.From(SHA256.HashData(clientCertificate)); + return ByteString.From(SHA256.HashData(clientCertificate.Span)); #else using SHA256 sha256 = SHA256.Create(); - return ByteString.From(sha256.ComputeHash(clientCertificate)); + return ByteString.From(sha256.ComputeHash(clientCertificate.ToArray())); #endif } diff --git a/src/Opc.Ua.Lds.Server/LdsServer.cs b/src/Opc.Ua.Lds.Server/LdsServer.cs index c2da324aaa..7713dc3c8d 100644 --- a/src/Opc.Ua.Lds.Server/LdsServer.cs +++ b/src/Opc.Ua.Lds.Server/LdsServer.cs @@ -493,8 +493,8 @@ protected virtual ServiceResult ValidateRegistration( new LocalizedText("ServerType is out of range.")); } - byte[]? certBytes = secureChannelContext?.ClientChannelCertificate; - if (certBytes == null || certBytes.Length == 0) + ByteString certBytes = secureChannelContext?.ClientChannelCertificate ?? default; + if (certBytes.Length == 0) { return new ServiceResult( StatusCodes.BadSecurityChecksFailed, @@ -503,7 +503,7 @@ protected virtual ServiceResult ValidateRegistration( try { - using var cert = Certificate.FromRawData(certBytes); + using var cert = Certificate.FromRawData(certBytes.Memory); IReadOnlyList applicationUris = X509Utils.GetApplicationUrisFromCertificate(cert); if (applicationUris.Count == 0) { diff --git a/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs b/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs index acf48853b4..d2692df51a 100644 --- a/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs +++ b/src/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs @@ -519,7 +519,7 @@ public override async ValueTask CloseSessionAsync( entry, endpoint.SecurityPolicyUri ?? string.Empty, endpoint.SecurityMode, - context.ChannelContext.ClientChannelCertificate.ToByteString()); + EmptyIfNull(context.ChannelContext.ClientChannelCertificate)); if (decision != RestoreDecision.Authorized) { m_logger.DistributedSessionRestoreRejected(TokenDigest(authenticationToken), decision); @@ -783,7 +783,7 @@ internal RestoreDecision AuthorizeRestore( } if (entry.OriginalClientChannelCertificate != - context.ChannelContext!.ClientChannelCertificate.ToByteString()) + context.ChannelContext!.ClientChannelCertificate) { return null; } @@ -912,7 +912,7 @@ private SharedSessionEntry BuildEntry( ClientCertificateChain = clientCertBlob, SecurityStateVersion = SharedSessionEntry.CurrentSecurityStateVersion, OriginalClientChannelCertificate = - context.ChannelContext!.ClientChannelCertificate.ToByteString(), + EmptyIfNull(context.ChannelContext!.ClientChannelCertificate), ClientUserId = null, ClientUserTokenType = UserTokenType.Anonymous, HasActivatedUserIdentity = false, @@ -1465,6 +1465,15 @@ private sealed record LivenessState(double SessionTimeout, DateTime MirroredCont private readonly ConcurrentDictionary m_tokensBySession = new(); private readonly ConcurrentDictionary m_mirrorLocks = new(); private readonly ConcurrentDictionary m_mirroredSequences = new(); + + /// + /// The channel certificate as the mirrored session state records it: empty, not + /// null, for a channel without a client certificate. + /// + private static ByteString EmptyIfNull(ByteString certificate) + { + return certificate.IsNull ? ByteString.Empty : certificate; + } } /// diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index 1055942831..85fb0babca 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -193,9 +193,10 @@ public ResourceIsolationOwner Classify( key = m_bindingKeys.GetValue(binding, value => new DerivedKey("user:" + Hash(Encoding.UTF8.GetBytes(value.ClientUserId!)))).Value; } - else if (secure && channelContext.ClientChannelCertificate is { Length: > 0 } certificate) + else if (secure && channelContext.ClientChannelCertificate.Length > 0) { - key = "application:" + Hash(certificate); + // classification-cache miss only + key = "application:" + Hash(channelContext.ClientChannelCertificate.ToArray()); } else { @@ -843,8 +844,9 @@ public ChannelSnapshot(SecureChannelContext context) m_channelId = context.SecureChannelId; SecurityPolicyUri = context.EndpointDescription?.SecurityPolicyUri; SecurityMode = context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid; + // ByteString is a read-only view, so the evidence cannot change after capture. m_certificate = SecurityPolicyUri != SecurityPolicies.None - ? context.ClientChannelCertificate?.AsSpan().ToArray() : null; + ? context.ClientChannelCertificate : default; m_peer = context.PeerAddress?.GetAddressBytes(); m_upstreamIdentity = context.UpstreamIdentity; } @@ -868,7 +870,7 @@ public bool Matches(SecureChannelContext context) SecurityPolicyUri == context.EndpointDescription?.SecurityPolicyUri && SecurityMode == (context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid) && (SecurityPolicyUri == SecurityPolicies.None || - m_certificate.AsSpan().SequenceEqual(context.ClientChannelCertificate.AsSpan())) && + m_certificate.Span.SequenceEqual(context.ClientChannelCertificate.Span)) && PeerMatches(context.PeerAddress) && ReferenceEquals(m_upstreamIdentity, context.UpstreamIdentity); } @@ -894,7 +896,7 @@ private bool PeerMatches(IPAddress? peer) } private readonly string m_channelId; - private readonly byte[]? m_certificate; + private readonly ByteString m_certificate; private readonly byte[]? m_peer; private readonly IUserIdentity? m_upstreamIdentity; } diff --git a/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs b/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs new file mode 100644 index 0000000000..f64496ff76 --- /dev/null +++ b/src/Opc.Ua.Server/Session/ChannelEvidenceExtensions.cs @@ -0,0 +1,48 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +namespace Opc.Ua.Server +{ + /// + /// Conversions of the read-only channel evidence in + /// for APIs that take arrays. + /// + internal static class ChannelEvidenceExtensions + { + /// + /// Returns a copy of the bytes, or null for a null byte string, matching + /// the nullable arrays the signature helpers expect. + /// + public static byte[]? ToArrayOrNull(this ByteString value) + { + return value.IsNull ? null : value.ToArray(); + } + } +} diff --git a/src/Opc.Ua.Server/Session/Session.cs b/src/Opc.Ua.Server/Session/Session.cs index 2ba782cdb3..c690876c7e 100644 --- a/src/Opc.Ua.Server/Session/Session.cs +++ b/src/Opc.Ua.Server/Session/Session.cs @@ -954,11 +954,11 @@ private void ValidateChannelBeforeActivate( byte[] clientNonceData = ClientNonce.ToArray(); byte[] dataToSign = securityPolicy.GetClientSignatureData( - context.ChannelContext.ChannelThumbprint, + context.ChannelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, m_serverCertificate.RawData, - context.ChannelContext.ServerChannelCertificate, - context.ChannelContext.ClientChannelCertificate, + context.ChannelContext.ServerChannelCertificate.ToArrayOrNull(), + context.ChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); if (!VerifyClientSignature(clientSignature!, dataToSign) && @@ -1010,11 +1010,11 @@ private bool VerifyClientSignatureOverServerCertificateChain( } byte[] dataToSign = securityPolicy.GetClientSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, [.. serverCertificateChainList], - channelContext.ServerChannelCertificate, - channelContext.ClientChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); return VerifyClientSignature(clientSignature, dataToSign); } @@ -1058,11 +1058,11 @@ private bool VerifyClientSignatureOverServerCertificate( byte[] serverCertificate) { byte[] dataToSign = securityPolicy.GetClientSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, serverCertificate, - channelContext.ServerChannelCertificate, - channelContext.ClientChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData); return VerifyClientSignature(clientSignature, dataToSign); } @@ -1513,12 +1513,12 @@ await token.DecryptAsync( byte[] clientNonceData = ClientNonce.ToArray(); byte[] dataToSign = securityPolicy.GetUserTokenSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, m_serverCertificate.RawData, - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), ClientCertificate?.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData, EndpointDescription.SecurityMode); @@ -1545,12 +1545,12 @@ await token.DecryptAsync( } dataToSign = securityPolicy.GetUserTokenSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), m_serverNonce.Data, [.. serverCertificateChainList], - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), ClientCertificate?.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonceData, EndpointDescription.SecurityMode); diff --git a/src/Opc.Ua.Server/Session/SessionManager.cs b/src/Opc.Ua.Server/Session/SessionManager.cs index e941aa68c0..9a7a5a3d4e 100644 --- a/src/Opc.Ua.Server/Session/SessionManager.cs +++ b/src/Opc.Ua.Server/Session/SessionManager.cs @@ -438,7 +438,10 @@ public virtual async ValueTask CreateSessionAsync( m_sessionActivationStates.Add( session, new SessionActivationState( - channelContext.ClientChannelCertificate.ToByteString(), + // empty rather than null for an unsecured channel, as before + channelContext.ClientChannelCertificate.IsNull + ? ByteString.Empty + : channelContext.ClientChannelCertificate, channelContext.EndpointDescription!.SecurityPolicyUri ?? SecurityPolicies.None, channelContext.EndpointDescription.SecurityMode)); @@ -748,7 +751,7 @@ private void ThrowIfClientNonceInUse(OperationContext context, ByteString client ((requiresClientCertificate && activationState.OriginalClientChannelCertificate.IsEmpty) || activationState.OriginalClientChannelCertificate != - channelContext.ClientChannelCertificate.ToByteString())) + channelContext.ClientChannelCertificate)) { throw new ServiceResultException( StatusCodes.BadSecurityChecksFailed, @@ -1286,21 +1289,23 @@ public bool TryGetSessionContext( SessionActivationState state; SessionBindingContext binding; ByteString originalClientChannelCertificate; - lock (m_bindingsLock) + + // Runs at admission and again at dispatch of every request. The session index + // and the activation-state table are concurrent collections, so the candidate + // snapshot is taken without the bindings lock; the check under the lock below + // confirms that the binding is still the committed one before it is returned. + if (authenticationToken.IsNull || + !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || + !m_sessionActivationStates.TryGetValue(currentSession, out SessionActivationState? currentState) || + currentState.IsCommitting || + currentState.BindingContext is not SessionBindingContext currentBinding) { - if (authenticationToken.IsNull || - !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || - !m_sessionActivationStates.TryGetValue(currentSession, out SessionActivationState? currentState) || - currentState.IsCommitting || - currentState.BindingContext is not SessionBindingContext currentBinding) - { - return false; - } - session = currentSession; - state = currentState; - binding = currentBinding; - originalClientChannelCertificate = state.OriginalClientChannelCertificate; + return false; } + session = currentSession; + state = currentState; + binding = currentBinding; + originalClientChannelCertificate = state.OriginalClientChannelCertificate; // Session diagnostics callbacks can query membership while holding a Session lock. // Probe Session state without the index lock, then check that the snapshot is still current. @@ -1313,7 +1318,7 @@ channelContext.EndpointDescription is not EndpointDescription endpoint || binding.SecurityMode != endpoint.SecurityMode || !string.Equals(binding.SecurityPolicyUri, endpoint.SecurityPolicyUri, StringComparison.Ordinal) || (binding.SecurityPolicyUri != SecurityPolicies.None && - originalClientChannelCertificate != channelContext.ClientChannelCertificate.ToByteString())) + originalClientChannelCertificate != channelContext.ClientChannelCertificate)) { return false; } @@ -1322,8 +1327,8 @@ channelContext.EndpointDescription is not EndpointDescription endpoint || { if (state.IsCommitting || !ReferenceEquals(state.BindingContext, binding) || - !m_sessions.TryGetValue(authenticationToken, out ISession? currentSession) || - !ReferenceEquals(currentSession, session)) + !m_sessions.TryGetValue(authenticationToken, out ISession? committedSession) || + !ReferenceEquals(committedSession, session)) { return false; } @@ -1969,8 +1974,7 @@ private void ValidateAnonymousSessionlessChannel( endpoint.EndpointUrl != null && Utils.IsUriHttpsScheme(endpoint.EndpointUrl)); if (requiresClientCertificate && - (secureChannelContext.ClientChannelCertificate == null || - secureChannelContext.ClientChannelCertificate.Length == 0)) + secureChannelContext.ClientChannelCertificate.Length == 0) { throw ServiceResultException.Create( StatusCodes.BadSecurityChecksFailed, diff --git a/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs b/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs index 5b1efa1911..0c9660fc41 100644 --- a/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs +++ b/src/Opc.Ua.Server/Session/SessionSecurityPolicyHelper.cs @@ -63,11 +63,11 @@ internal static class SessionSecurityPolicyHelper SecureChannelContext channelContext = context.ChannelContext!; byte[] dataToSign = securityPolicy!.GetServerSignatureData( - channelContext.ChannelThumbprint, + channelContext.ChannelThumbprint.ToArrayOrNull(), clientNonce.ToArray(), - channelContext.ServerChannelCertificate, + channelContext.ServerChannelCertificate.ToArrayOrNull(), parsedClientCertificate.RawData, - channelContext.ClientChannelCertificate, + channelContext.ClientChannelCertificate.ToArrayOrNull(), serverNonce.ToArray()); return policies.CreateSignatureData( diff --git a/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs new file mode 100644 index 0000000000..28af03d742 --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Stack/Server/SecureChannelContextTests.cs @@ -0,0 +1,89 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using NUnit.Framework; + +namespace Opc.Ua.Core.Tests.Stack.Server +{ + /// + /// Tests for the read-only channel evidence carried by . + /// + [TestFixture] + [Category("SecureChannelContext")] + [Parallelizable] + public class SecureChannelContextTests + { + /// + /// Missing evidence stays null rather than becoming empty: the session signature + /// helpers hash an empty certificate but skip a missing one. + /// + [Test] + public void MissingEvidenceIsNullNotEmpty() + { + var context = new SecureChannelContext("1", null, RequestEncoding.Binary); + + Assert.That(context.ClientChannelCertificate.IsNull, Is.True); + Assert.That(context.ServerChannelCertificate.IsNull, Is.True); + Assert.That(context.ChannelThumbprint.IsNull, Is.True); + } + + /// + /// Empty evidence stays empty and is not reported as null. + /// + [Test] + public void EmptyEvidenceIsNotNull() + { + var context = new SecureChannelContext("1", null, RequestEncoding.Binary, [], [], []); + + Assert.That(context.ClientChannelCertificate.IsNull, Is.False); + Assert.That(context.ClientChannelCertificate.IsEmpty, Is.True); + } + + /// + /// The evidence is a view of the transport's buffer, so contexts created for the + /// requests of one channel share it without copying it. + /// + [Test] + public void EvidenceWrapsTheTransportBufferWithoutCopying() + { + byte[] certificate = [1, 2, 3, 4]; + + var first = new SecureChannelContext("1", null, RequestEncoding.Binary, certificate); + var second = new SecureChannelContext("1", null, RequestEncoding.Binary, certificate); + + Assert.That(first.ClientChannelCertificate.Span.ToArray(), Is.EqualTo(certificate)); + Assert.That( + first.ClientChannelCertificate.Span.Overlaps(second.ClientChannelCertificate.Span), + Is.True); + Assert.That(first.ClientChannelCertificate, Is.EqualTo(second.ClientChannelCertificate)); + } + } +} diff --git a/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs b/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs index aa43a4f45e..0113cd8d36 100644 --- a/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs +++ b/tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs @@ -112,7 +112,7 @@ public async Task SessionIsMirroredEncryptedOnActivateAndRemovedOnCloseAsync() Assert.That(entry.HasActivatedUserIdentity, Is.True); Assert.That( entry.OriginalClientChannelCertificate, - Is.EqualTo(context.ClientChannelCertificate.ToByteString())); + Is.EqualTo(context.ClientChannelCertificate)); // Encrypted at rest: a protector with a different key fails closed. using var wrongKey = new AesCbcHmacRecordProtector(MakeKey(2)); diff --git a/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs b/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs index 1b25d9e7e7..79c14cd08b 100644 --- a/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs +++ b/tests/Opc.Ua.Server.TestFramework/ServerFixtureUtils.cs @@ -188,11 +188,11 @@ public static class ServerFixtureUtils createSessionResponse.ServerCertificate, server.MessageContext.Telemetry); byte[] dataToSign = securityPolicy.GetClientSignatureData( - secureChannelContext.ChannelThumbprint, + secureChannelContext.ChannelThumbprint.ToArrayOrNull(), createSessionResponse.ServerNonce.ToArray(), serverCertificateChain[0].RawData, - secureChannelContext.ServerChannelCertificate, - secureChannelContext.ClientChannelCertificate, + secureChannelContext.ServerChannelCertificate.ToArrayOrNull(), + secureChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonce.ToArray()); clientSignature = SecurityPolicies.Default.CreateSignatureData( securityPolicy, diff --git a/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs b/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs index 21b3131816..2968e11092 100644 --- a/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs +++ b/tests/Opc.Ua.Server.Tests/ResourceIsolation/RuntimeResourceIsolationTests.cs @@ -1015,7 +1015,7 @@ public void ClassifierCannotObserveCertificateOnNoneChannel() var classifier = new Mock(MockBehavior.Strict); ResourceIsolationIdentity identity = default; classifier.Setup(c => c.TryClassify( - It.Is(c => c.ClientChannelCertificate == null), + It.Is(c => c.ClientChannelCertificate.IsNull), null, out identity)).Returns(false); classifier.Setup(c => c.TryClassifyIngress(It.IsAny(), out identity)).Returns(false); using DefaultServerResourceIsolationProvider provider = CreateProvider(Options(), classifier.Object); diff --git a/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs b/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs index ea83026193..682189b25d 100644 --- a/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/Session/EphemeralNonceLifetimeRegressionTests.cs @@ -582,8 +582,8 @@ public NonceHarness(Action? beforeSignature = null, bool forceRsa beforeSignature?.Invoke(this); ClientSignature = SecurityPolicies.Default.CreateSignatureData( policy, ClientCertificate, policy.GetClientSignatureData( - channel.ChannelThumbprint, ServerNonce.Data, ServerCertificate.RawData, - channel.ServerChannelCertificate, channel.ClientChannelCertificate, clientNonce.Data)); + channel.ChannelThumbprint.ToArrayOrNull(), ServerNonce.Data, ServerCertificate.RawData, + channel.ServerChannelCertificate.ToArrayOrNull(), channel.ClientChannelCertificate.ToArrayOrNull(), clientNonce.Data)); } catch { diff --git a/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs b/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs index 8f0152e416..04dcf1dd8d 100644 --- a/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs +++ b/tests/Opc.Ua.Server.Tests/SessionRestoreTests.cs @@ -654,7 +654,7 @@ private async ValueTask CreateRestoredSessionAsync( context.ChannelContext!.EndpointDescription!; SetRestoredSessionTransferSecurityState( session, - context.ChannelContext.ClientChannelCertificate.ToByteString(), + context.ChannelContext.ClientChannelCertificate, endpoint.SecurityPolicyUri ?? SecurityPolicies.None, endpoint.SecurityMode, m_behavior == RestoreBehavior.RestoreWithDifferentOwner diff --git a/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs b/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs index 3577087b24..00f73bc469 100644 --- a/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs +++ b/tests/Opc.Ua.Server.Tests/SessionSecurityTests.cs @@ -528,12 +528,12 @@ public async Task CertificateUserPolicyVerifiesProofOnNoneChannelAsync(string pr SecurityPolicyInfo policy = SecurityPolicies.Default.GetInfo(SecurityPolicies.Basic256Sha256)!; SecureChannelContext channel = created.Context.ChannelContext!; byte[] dataToSign = policy.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray()); SignatureData userSignature = proof switch { @@ -607,21 +607,21 @@ public async Task EnhancedCertificateUserPolicyUsesNoneVariantOnNoneChannelAsync SecureChannelContext channel = created.Context.ChannelContext!; byte[] dataToSign = noneVariant ? policy!.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray(), MessageSecurityMode.None) : policy!.GetUserTokenSignatureData( - channel.ChannelThumbprint, + channel.ChannelThumbprint.ToArrayOrNull(), created.ServerNonce.ToArray(), m_serverCertificate.RawData, - channel.ServerChannelCertificate, + channel.ServerChannelCertificate.ToArrayOrNull(), m_clientCertificate.RawData, - channel.ClientChannelCertificate, + channel.ClientChannelCertificate.ToArrayOrNull(), created.ClientNonce.ToArray()); SignatureData userSignature = await SecurityPolicies.Default.CreateSignatureDataAsync( policy, @@ -1459,11 +1459,11 @@ private SignatureData CreateClientSignature( SecurityPolicyInfo securityPolicy = SecurityPolicies.Default.GetInfo( context.ChannelContext!.EndpointDescription!.SecurityPolicyUri!)!; byte[] dataToSign = securityPolicy.GetClientSignatureData( - context.ChannelContext.ChannelThumbprint, + context.ChannelContext.ChannelThumbprint.ToArrayOrNull(), serverNonce.ToArray(), m_serverCertificate.RawData, - context.ChannelContext.ServerChannelCertificate, - context.ChannelContext.ClientChannelCertificate, + context.ChannelContext.ServerChannelCertificate.ToArrayOrNull(), + context.ChannelContext.ClientChannelCertificate.ToArrayOrNull(), clientNonce.ToArray()); return SecurityPolicies.Default.CreateSignatureData( securityPolicy, From a0d9b9e7a42442c3e3cf32a8ae70ff06eab5ff6d Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:08 +0200 Subject: [PATCH 05/19] Tests: cover ObjectPool, the per-request system context cache and the translation snapshot ObjectPool: reuse, bounded retention, the per-processor cap, no shared rental under concurrent Get/Return, and no lost capacity after concurrent use. OperationContext.GetSystemContext: one copy per template, never another node manager's copy when templates alternate, and concurrent lookups bound to the same operation. ResourceManager: updates and new locales are visible after a snapshot was built, and concurrent readers see only complete translations. Co-Authored-By: Claude Opus 5.5 --- .../Types/Utils/ObjectPoolTests.cs | 196 ++++++++++++++++++ ...OperationContextSystemContextCacheTests.cs | 140 +++++++++++++ .../ResourceManagerSnapshotTests.cs | 134 ++++++++++++ 3 files changed, 470 insertions(+) create mode 100644 tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs create mode 100644 tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs create mode 100644 tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs diff --git a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs new file mode 100644 index 0000000000..66584127de --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs @@ -0,0 +1,196 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace Opc.Ua.Core.Tests.Types.UtilsTests +{ + /// + /// Tests for the bounded lock-free . + /// + [TestFixture] + [Category("Utils")] + [Parallelizable] + public class ObjectPoolTests + { + /// + /// A returned object is handed out again before a new one is created. + /// + [Test] + public void ReturnedObjectIsReused() + { + int created = 0; + var pool = new ObjectPool(() => { created++; return new object(); }, 4); + + object first = pool.Get(); + pool.Return(first); + object second = pool.Get(); + + Assert.That(second, Is.SameAs(first)); + Assert.That(created, Is.EqualTo(1)); + } + + /// + /// The pool retains at most its capacity; objects returned beyond it are dropped. + /// + [Test] + public void RetainsAtMostTheCapacity() + { + int capacity = Math.Min(3, Environment.ProcessorCount * 4); + var pool = new ObjectPool(() => new object(), capacity); + var rented = new List(); + for (int ii = 0; ii < capacity + 5; ii++) + { + rented.Add(pool.Get()); + } + foreach (object item in rented) + { + pool.Return(item); + } + + var reused = new HashSet(); + for (int ii = 0; ii < capacity + 5; ii++) + { + object item = pool.Get(); + if (rented.Contains(item)) + { + reused.Add(item); + } + } + + Assert.That(reused, Has.Count.EqualTo(capacity)); + } + + /// + /// The capacity is capped at four objects per processor. + /// + [Test] + public void CapacityIsCappedByProcessorCount() + { + int cap = Environment.ProcessorCount * 4; + var pool = new ObjectPool(() => new object(), cap * 10); + var rented = new List(); + for (int ii = 0; ii < cap * 2; ii++) + { + rented.Add(pool.Get()); + } + foreach (object item in rented) + { + pool.Return(item); + } + + int reused = 0; + for (int ii = 0; ii < cap * 2; ii++) + { + if (rented.Contains(pool.Get())) + { + reused++; + } + } + + Assert.That(reused, Is.EqualTo(cap)); + } + + /// + /// Under concurrent Get and Return an object is never held by two renters at once. + /// + [Test] + public void ConcurrentRentalsNeverShareAnObject() + { + var pool = new ObjectPool(() => new Rental(), 64); + int duplicates = 0; + + Parallel.For(0, Environment.ProcessorCount * 2, _ => + { + for (int ii = 0; ii < 20000; ii++) + { + Rental rental = pool.Get(); + if (Interlocked.Exchange(ref rental.InUse, 1) != 0) + { + Interlocked.Increment(ref duplicates); + } + Thread.SpinWait(ii % 8); + Volatile.Write(ref rental.InUse, 0); + pool.Return(rental); + } + }); + + Assert.That(duplicates, Is.Zero); + } + + /// + /// Concurrent use does not lose capacity: once quiet, the pool still serves its retained + /// objects without creating new ones. + /// + [Test] + public void ConcurrentUseDoesNotLoseCapacity() + { + int created = 0; + var pool = new ObjectPool( + () => + { + Interlocked.Increment(ref created); + return new object(); + }, + Environment.ProcessorCount * 4); + var seen = new ConcurrentDictionary(); + + Parallel.For(0, Environment.ProcessorCount, _ => + { + for (int ii = 0; ii < 10000; ii++) + { + object item = pool.Get(); + seen.TryAdd(item, true); + pool.Return(item); + } + }); + + int createdAfterLoad = Volatile.Read(ref created); + var rented = new List(); + for (int ii = 0; ii < Math.Min(createdAfterLoad, Environment.ProcessorCount * 4); ii++) + { + rented.Add(pool.Get()); + } + + Assert.That(Volatile.Read(ref created), Is.EqualTo(createdAfterLoad)); + Assert.That(rented, Is.All.Matches(seen.ContainsKey)); + } + + private sealed class Rental + { + public int InUse; + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs b/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs new file mode 100644 index 0000000000..01ab911217 --- /dev/null +++ b/tests/Opc.Ua.Server.Tests/OperationContextSystemContextCacheTests.cs @@ -0,0 +1,140 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System.Linq; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests +{ + /// + /// Tests for the per-request system context copy cached on , + /// which node managers reuse for the per-node callbacks of one request. + /// + [TestFixture] + [Category("OperationContext")] + [SetCulture("en-us")] + [SetUICulture("en-us")] + [Parallelizable] + public class OperationContextSystemContextCacheTests + { + /// + /// Repeated lookups with the same template return the same copy, bound to the operation. + /// + [Test] + public void SameTemplateReturnsOneCopyBoundToTheOperation() + { + ServerSystemContext template = CreateTemplate(); + using OperationContext operation = CreateOperation(); + + ServerSystemContext first = operation.GetSystemContext(template); + ServerSystemContext second = operation.GetSystemContext(template); + + Assert.That(second, Is.SameAs(first)); + Assert.That(first, Is.Not.SameAs(template)); + Assert.That(first.OperationContext, Is.SameAs(operation)); + } + + /// + /// A copy made from one node manager's template is never handed to another node manager. + /// + [Test] + public void SwitchingTemplatesNeverReturnsAnotherManagersCopy() + { + ServerSystemContext templateA = CreateTemplate(systemHandle: "A"); + ServerSystemContext templateB = CreateTemplate(systemHandle: "B"); + using OperationContext operation = CreateOperation(); + + ServerSystemContext a1 = operation.GetSystemContext(templateA); + ServerSystemContext b1 = operation.GetSystemContext(templateB); + ServerSystemContext a2 = operation.GetSystemContext(templateA); + + Assert.That(a1.SystemHandle, Is.EqualTo("A")); + Assert.That(b1.SystemHandle, Is.EqualTo("B")); + Assert.That(a2.SystemHandle, Is.EqualTo("A")); + Assert.That(b1, Is.Not.SameAs(a1)); + Assert.That(b1.OperationContext, Is.SameAs(operation)); + Assert.That(a2.OperationContext, Is.SameAs(operation)); + } + + /// + /// Concurrent lookups with alternating templates always get a copy of the template they + /// passed, bound to the same operation. + /// + [Test] + public void ConcurrentLookupsAlwaysReturnACopyOfTheRequestedTemplate() + { + ServerSystemContext[] templates = + [ + CreateTemplate(systemHandle: "A"), + CreateTemplate(systemHandle: "B"), + CreateTemplate(systemHandle: "C") + ]; + using OperationContext operation = CreateOperation(); + + int mismatches = 0; + Parallel.For(0, 30000, ii => + { + ServerSystemContext template = templates[ii % templates.Length]; + ServerSystemContext copy = operation.GetSystemContext(template); + if (!Equals(copy.SystemHandle, template.SystemHandle) || + !ReferenceEquals(copy.OperationContext, operation)) + { + System.Threading.Interlocked.Increment(ref mismatches); + } + }); + + Assert.That(mismatches, Is.Zero); + Assert.That( + templates.Select(t => operation.GetSystemContext(t).SystemHandle), + Is.EqualTo(new object[] { "A", "B", "C" })); + } + + private static ServerSystemContext CreateTemplate(object? systemHandle = null) + { + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(NUnitTelemetryContext.Create()); + server.Setup(s => s.NamespaceUris).Returns(new NamespaceTable()); + server.Setup(s => s.ServerUris).Returns(new StringTable()); + return new ServerSystemContext(server.Object) { SystemHandle = systemHandle }; + } + + private static OperationContext CreateOperation() + { + return new OperationContext( + new RequestHeader(), + null, + RequestType.Read, + RequestLifetime.None); + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs new file mode 100644 index 0000000000..7b0d53add2 --- /dev/null +++ b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs @@ -0,0 +1,134 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests +{ + /// + /// Tests that the lock-free translation snapshot of is rebuilt + /// after every change, so translations always reflect the tables as written. + /// + [TestFixture] + [Category("ResourceManager")] + [Parallelizable] + [SetCulture("en-us")] + [SetUICulture("en-us")] + public class ResourceManagerSnapshotTests + { + /// + /// A translation read after an update, and after a new locale is added, sees the change. + /// + [Test] + public void UpdatesAndNewLocalesAreVisibleAfterASnapshotWasBuilt() + { + using ResourceManager resourceManager = CreateResourceManager(); + resourceManager.Add("greeting", "de-DE", "Hallo"); + + // builds the snapshot + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Hallo")); + + resourceManager.Add("greeting", "de-DE", "Servus"); + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Servus")); + + resourceManager.Add("greeting", "fr-FR", "Bonjour"); + Assert.That(Translate(resourceManager, "fr-FR"), Is.EqualTo("Bonjour")); + Assert.That(Translate(resourceManager, "de-DE"), Is.EqualTo("Servus")); + Assert.That(resourceManager.GetAvailableLocales(), Does.Contain("fr-FR")); + } + + /// + /// Readers running while translations are added never fail, never see a partially + /// written table, and see every translation once the writer is done. + /// + [Test] + public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() + { + using ResourceManager resourceManager = CreateResourceManager(); + const int keyCount = 500; + using var cts = new CancellationTokenSource(); + int wrongTexts = 0; + + Task[] readers = new Task[4]; + for (int r = 0; r < readers.Length; r++) + { + readers[r] = Task.Run(() => + { + while (!cts.IsCancellationRequested) + { + for (int ii = 0; ii < keyCount; ii += 37) + { + string key = "key" + ii.ToString(CultureInfo.InvariantCulture); + LocalizedText text = resourceManager.Translate(["de-DE"], key, "default"); + // either not added yet (default) or the complete translation + if (text.Text != "default" && text.Text != "Wert" + ii.ToString(CultureInfo.InvariantCulture)) + { + Interlocked.Increment(ref wrongTexts); + } + } + } + }); + } + + for (int ii = 0; ii < keyCount; ii++) + { + resourceManager.Add( + "key" + ii.ToString(CultureInfo.InvariantCulture), + "de-DE", + "Wert" + ii.ToString(CultureInfo.InvariantCulture)); + } + cts.Cancel(); + Assert.That(Task.WaitAll(readers, 30000), Is.True); + + Assert.That(wrongTexts, Is.Zero); + for (int ii = 0; ii < keyCount; ii++) + { + string key = "key" + ii.ToString(CultureInfo.InvariantCulture); + Assert.That( + resourceManager.Translate(["de-DE"], key, "default").Text, + Is.EqualTo("Wert" + ii.ToString(CultureInfo.InvariantCulture))); + } + } + + private static ResourceManager CreateResourceManager() + { + return new ResourceManager(new ApplicationConfiguration(NUnitTelemetryContext.Create())); + } + + private static string? Translate(ResourceManager resourceManager, string locale) + { + return resourceManager.Translate([locale], "greeting", "Hello").Text; + } + } +} From 840b264142ec56463e9e42f601b6b25000d58bad Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:01:09 +0200 Subject: [PATCH 06/19] perf: follow repository conventions in the harness and trace analyzer Add the OPC Foundation license header, drop the per-project Nullable, warnings-as-errors and analysis-level overrides and fix the diagnostics instead, remove comment section dividers, and record latency samples in a ConcurrentBag instead of locking List instances. The only suppression left is a documented CA2000 where the session takes ownership of a subscription. Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/Program.cs | 1144 ++++++++++------- .../ServerLoadHarness.csproj | 11 +- perf/TraceAnalyzer/Program.cs | 384 ++++-- perf/TraceAnalyzer/TraceAnalyzer.csproj | 6 - 4 files changed, 906 insertions(+), 639 deletions(-) diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index a1bc5d8c73..320d3807ba 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -1,647 +1,821 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + // Server load / profiling harness. Not part of the product; see perf/README.md. // Modes: // server --port P [--diag 0|1] [--audit 0|1] [--threads N] // Starts a ReferenceServer and reads commands from stdin: mark | report | quit // run --scenario read|write|sub|browse [--sessions N] [--nodes K] [--inflight I] // [--duration S] [--warmup S] [--security none|sign|encrypt] [--diag 0|1] [--audit 0|1] -// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] +// [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] [--out file] // Spawns the server child, drives the load and prints a report of client latency // and server-side CPU / GC / allocation deltas for the measurement window. +using System; +using System.Collections; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; +using System.IO; +using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Opc.Ua; using Opc.Ua.Client; using Opc.Ua.Client.TestFramework; using Opc.Ua.Configuration; using Opc.Ua.Server.TestFramework; using Quickstarts.ReferenceServer; -internal static class Program +namespace Opc.Ua.Perf.ServerLoadHarness { - private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); - - private static string Arg(string name, string def) => s_args.TryGetValue(name, out string? v) ? v : def; - private static int ArgInt(string name, int def) => int.Parse(Arg(name, def.ToString(CultureInfo.InvariantCulture)), CultureInfo.InvariantCulture); - - public static async Task Main(string[] args) + /// + /// Entry point of the harness. + /// + public static class Program { - for (int i = 1; i < args.Length; i++) + private static readonly Dictionary s_args = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Runs the server or the load driver. + /// + public static async Task Main(string[] args) { - if (args[i].StartsWith("--", StringComparison.Ordinal)) + for (int i = 1; i < args.Length; i++) { - string key = args[i][2..]; - string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) ? args[++i] : "1"; - s_args[key] = value; + if (args[i].StartsWith("--", StringComparison.Ordinal)) + { + string key = args[i][2..]; + string value = i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal) + ? args[++i] + : "1"; + s_args[key] = value; + } } + string mode = args.Length > 0 ? args[0] : "run"; + return mode == "server" + ? await RunServerAsync().ConfigureAwait(false) + : await RunDriverAsync(args).ConfigureAwait(false); } - string mode = args.Length > 0 ? args[0] : "run"; - return mode switch - { - "server" => await RunServerAsync().ConfigureAwait(false), - _ => await RunDriverAsync(args).ConfigureAwait(false) - }; - } - - // ------------------------------------------------------------------ server - - private static ITelemetryContext CreateTelemetry(LogLevel level) => - DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); - private static async Task RunServerAsync() - { - int port = ArgInt("port", 62555); - bool diag = ArgInt("diag", 0) != 0; - bool audit = ArgInt("audit", 0) != 0; - ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); - - string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); - var application = new ApplicationInstance(telemetry) - { - ApplicationName = "PerfReferenceServer", - ApplicationType = ApplicationType.Server - }; - string endpointUrl = $"opc.tcp://localhost:{port}/PerfReferenceServer"; - ArrayOf certs = ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( - "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", CertificateStoreType.Directory, pkiRoot); - ApplicationConfiguration config = await application - .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") - .SetMaxByteStringLength(4 * 1024 * 1024) - .SetMaxArrayLength(1024 * 1024) - .SetMaxMessageSize(16 * 1024 * 1024) - .AsServer([endpointUrl]) - .AddUnsecurePolicyNone() - .AddSignPolicies() - .AddSignAndEncryptPolicies() - .SetMaxChannelCount(2100) - .SetDiagnosticsEnabled(diag) - .SetAuditingEnabled(audit) - .SetShutdownDelay(0) - .AddSecurityConfiguration(certs, pkiRoot) - .SetAutoAcceptUntrustedCertificates(true) - .CreateAsync().ConfigureAwait(false); - config.ServerConfiguration!.MaxSessionCount = 2000; - config.ServerConfiguration.MaxSubscriptionCount = 20000; - config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; - config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); - config.ServerConfiguration.MinRequestThreadCount = 50; - - if (!await application.CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime).ConfigureAwait(false)) + internal static string Arg(string name, string defaultValue) { - Console.WriteLine("ERROR cert"); - return 1; + return s_args.TryGetValue(name, out string? value) ? value : defaultValue; } - var server = new ReferenceServer(telemetry); - server.TransportBindings = TestTransportBindings.WithAllSchemes(); - await application.StartAsync(server).ConfigureAwait(false); - Console.WriteLine($"READY {endpointUrl} {Environment.ProcessId}"); + internal static int ArgInt(string name, int defaultValue) + { + return int.Parse( + Arg(name, defaultValue.ToString(CultureInfo.InvariantCulture)), + CultureInfo.InvariantCulture); + } - ServerStats mark = ServerStats.Capture(); - string? line; - while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) + internal static List GetNodes(ISession session, int count, Type? onlyType = null) { - switch (line.Trim()) - { - case "mark": - GC.Collect(); - GC.WaitForPendingFinalizers(); - mark = ServerStats.Capture(); - Console.WriteLine("MARKED"); - break; - case "report": - Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); - break; - case "quit": - await server.StopAsync().ConfigureAwait(false); - return 0; + var result = new List(); + foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + { + if (onlyType != null && type != onlyType) + { + continue; + } + foreach (ExpandedNodeId id in ids) + { + if (result.Count == count) + { + return result; + } + result.Add(ExpandedNodeId.ToNodeId(id, session.NamespaceUris)); + } } + return result; } - return 0; - } - private sealed record ServerStats(long Alloc, int G0, int G1, int G2, TimeSpan Pause, TimeSpan Cpu, long Contention, long Stamp, int Threads, long WorkingSet, long Heap) - { - public static ServerStats Capture() + private static ITelemetryContext CreateTelemetry(LogLevel level) { - using var p = Process.GetCurrentProcess(); - return new ServerStats( - GC.GetTotalAllocatedBytes(true), GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), - GC.GetTotalPauseDuration(), p.TotalProcessorTime, Monitor.LockContentionCount, Stopwatch.GetTimestamp(), - ThreadPool.ThreadCount, p.WorkingSet64, GC.GetGCMemoryInfo().HeapSizeBytes); + return DefaultTelemetry.Create(b => b.SetMinimumLevel(level).AddSimpleConsole(o => o.SingleLine = true)); } - public string Delta(ServerStats m) + private static async Task RunServerAsync() { - double sec = Stopwatch.GetElapsedTime(m.Stamp, Stamp).TotalSeconds; - return string.Create(CultureInfo.InvariantCulture, - $"sec={sec:F2} allocMB={(Alloc - m.Alloc) / 1048576.0:F1} gen0={G0 - m.G0} gen1={G1 - m.G1} gen2={G2 - m.G2} pauseMs={(Pause - m.Pause).TotalMilliseconds:F0} cpuMs={(Cpu - m.Cpu).TotalMilliseconds:F0} cpuCores={(Cpu - m.Cpu).TotalSeconds / sec:F2} contention={Contention - m.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + var application = new ApplicationInstance(telemetry) + { + ApplicationName = "PerfReferenceServer", + ApplicationType = ApplicationType.Server + }; + try + { + return await RunServerAsync(application, telemetry).ConfigureAwait(false); + } + finally + { + await application.DisposeAsync().ConfigureAwait(false); + } } - } - // ------------------------------------------------------------------ driver + private static async Task RunServerAsync(ApplicationInstance application, ITelemetryContext telemetry) + { + int port = ArgInt("port", 62555); + bool diagnostics = ArgInt("diag", 0) != 0; + bool auditing = ArgInt("audit", 0) != 0; + string pkiRoot = Path.Combine(Path.GetTempPath(), "uaperf-pki-server"); + string endpointUrl = FormattableString.Invariant($"opc.tcp://localhost:{port}/PerfReferenceServer"); + ArrayOf certificates = + ApplicationConfigurationBuilder.CreateDefaultApplicationCertificates( + "CN=PerfReferenceServer, O=OPC Foundation, DC=localhost", + CertificateStoreType.Directory, + pkiRoot); + ApplicationConfiguration config = await application + .Build("urn:localhost:UA:PerfReferenceServer", "uri:opcfoundation.org:PerfReferenceServer") + .SetMaxByteStringLength(4 * 1024 * 1024) + .SetMaxArrayLength(1024 * 1024) + .SetMaxMessageSize(16 * 1024 * 1024) + .AsServer([endpointUrl]) + .AddUnsecurePolicyNone() + .AddSignPolicies() + .AddSignAndEncryptPolicies() + .SetMaxChannelCount(2100) + .SetDiagnosticsEnabled(diagnostics) + .SetAuditingEnabled(auditing) + .SetShutdownDelay(0) + .AddSecurityConfiguration(certificates, pkiRoot) + .SetAutoAcceptUntrustedCertificates(true) + .CreateAsync() + .ConfigureAwait(false); + config.ServerConfiguration!.MaxSessionCount = 2000; + config.ServerConfiguration.MaxSubscriptionCount = 20000; + config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; + config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); + config.ServerConfiguration.MinRequestThreadCount = 50; + + if (!await application + .CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime) + .ConfigureAwait(false)) + { + Console.WriteLine("ERROR cert"); + return 1; + } - private static async Task RunDriverAsync(string[] args) - { - string scenario = Arg("scenario", "read"); - int sessionsCount = ArgInt("sessions", 20); - int durationSec = ArgInt("duration", 20); - int warmupSec = ArgInt("warmup", 10); - int port = ArgInt("port", 62555 + Environment.ProcessId % 1000); - string security = Arg("security", "none"); - - string exe = Environment.ProcessPath!; - var psi = new ProcessStartInfo(exe, $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}") - { - RedirectStandardInput = true, - RedirectStandardOutput = true, - UseShellExecute = false - }; - foreach (System.Collections.DictionaryEntry kv in Environment.GetEnvironmentVariables()) - { - string k = (string)kv.Key; - if (k.StartsWith("SERVER_", StringComparison.Ordinal)) + using var server = new ReferenceServer(telemetry) + { + TransportBindings = TestTransportBindings.WithAllSchemes() + }; + await application.StartAsync(server).ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant($"READY {endpointUrl} {Environment.ProcessId}")); + + ServerStats mark = ServerStats.Capture(); + string? line; + while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) { - psi.Environment[k[7..]] = (string?)kv.Value; + switch (line.Trim()) + { + case "mark": + GC.Collect(); + GC.WaitForPendingFinalizers(); + mark = ServerStats.Capture(); + Console.WriteLine("MARKED"); + break; + case "report": + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + break; + case "quit": + await server.StopAsync().ConfigureAwait(false); + return 0; + } } + return 0; } - using Process server = Process.Start(psi)!; - var serverLines = new BlockingCollection(); - _ = Task.Run(() => + + private static async Task RunDriverAsync(string[] args) { - string? l; - while ((l = server.StandardOutput.ReadLine()) != null) + string scenario = Arg("scenario", "read"); + int sessionCount = ArgInt("sessions", 20); + int durationSeconds = ArgInt("duration", 20); + int warmupSeconds = ArgInt("warmup", 10); + int port = ArgInt("port", 62555 + (Environment.ProcessId % 1000)); + string security = Arg("security", "none"); + + var startInfo = new ProcessStartInfo( + Environment.ProcessPath!, + FormattableString.Invariant( + $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}")) { - if (l.StartsWith("READY", StringComparison.Ordinal) || l.StartsWith("REPORT", StringComparison.Ordinal) || l.StartsWith("MARKED", StringComparison.Ordinal) || l.StartsWith("ERROR", StringComparison.Ordinal)) + RedirectStandardInput = true, + RedirectStandardOutput = true, + UseShellExecute = false + }; + foreach (DictionaryEntry variable in Environment.GetEnvironmentVariables()) + { + string name = (string)variable.Key; + if (name.StartsWith("SERVER_", StringComparison.Ordinal)) { - serverLines.Add(l); + startInfo.Environment[name[7..]] = (string?)variable.Value; } - else if (s_args.ContainsKey("verbose")) + } + + using Process server = Process.Start(startInfo)!; + using var serverLines = new BlockingCollection(); + _ = Task.Run(() => PumpServerOutput(server, serverLines)); + string ready = serverLines.Take(); + if (!ready.StartsWith("READY", StringComparison.Ordinal)) + { + Console.WriteLine(ready); + return 1; + } + string url = ready.Split(' ')[1]; + Console.WriteLine(FormattableString.Invariant($"server pid={server.Id} url={url}")); + + ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); + using var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; + await client + .LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient") + .ConfigureAwait(false); + string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; + MessageSecurityMode mode = security switch + { + "sign" => MessageSecurityMode.Sign, + "encrypt" => MessageSecurityMode.SignAndEncrypt, + _ => MessageSecurityMode.None + }; + ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); + var selected = new List(); + foreach (EndpointDescription endpoint in endpoints) + { + if (endpoint.SecurityPolicyUri == policy && endpoint.SecurityMode == mode) { - Console.WriteLine("[server] " + l); + selected.Add(endpoint); } } - serverLines.CompleteAdding(); - }); - string ready = serverLines.Take(); - if (!ready.StartsWith("READY", StringComparison.Ordinal)) - { - Console.WriteLine(ready); - return 1; + + var sessions = new ISession[sessionCount]; + var connectTime = Stopwatch.StartNew(); + await Parallel.ForAsync( + 0, + sessionCount, + new ParallelOptions { MaxDegreeOfParallelism = 32 }, + async (i, ct) => sessions[i] = await client + .ConnectAsync(new Uri(url), policy, selected) + .ConfigureAwait(false)) + .ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"connected {sessionCount} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); + + Workload workload = scenario == "sub" + ? new SubscribeWorkload( + sessions, + ArgInt("nodes", 100), + ArgInt("subs", 5), + ArgInt("pub", 100), + ArgInt("write-interval", 100)) + : new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)); + await workload.SetupAsync().ConfigureAwait(false); + + using var stop = new CancellationTokenSource(); + Task run = workload.RunAsync(stop.Token); + await Task.Delay(TimeSpan.FromSeconds(warmupSeconds)).ConfigureAwait(false); + + using Process? tracer = StartTracer(server.Id, durationSeconds, out string? trace); + if (tracer != null) + { + await Task.Delay(2000).ConfigureAwait(false); + } + + await server.StandardInput.WriteLineAsync("mark").ConfigureAwait(false); + Take(serverLines, "MARKED"); + workload.ResetStats(); + var window = Stopwatch.StartNew(); + await Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ConfigureAwait(false); + await server.StandardInput.WriteLineAsync("report").ConfigureAwait(false); + string report = Take(serverLines, "REPORT")[7..]; + string clientReport = workload.Report(window.Elapsed.TotalSeconds); + await stop.CancelAsync().ConfigureAwait(false); + try + { + await run.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // the workload stops by cancellation + } + + if (tracer != null) + { + await tracer.WaitForExitAsync().ConfigureAwait(false); + Console.WriteLine("trace: " + trace); + } + + var result = new StringBuilder(); + result.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionCount} ") + .Append(string.Join(' ', args.Skip(1))) + .Append('\n') + .Append(" client: ").Append(clientReport).Append('\n') + .Append(" server: ").Append(report).Append('\n'); + double operations = workload.Operations; + if (operations > 0) + { + (double allocatedMb, double cpuMs) = ParseServerReport(report); + result.Append( + CultureInfo.InvariantCulture, + $" per-op: allocB={allocatedMb * 1048576 / operations:F0} cpuUs={cpuMs * 1000 / operations:F1} ({workload.OperationName})\n"); + } + Console.Write(result.ToString()); + if (s_args.TryGetValue("out", out string? outFile)) + { + await File.AppendAllTextAsync(outFile, result.ToString()).ConfigureAwait(false); + } + + await Parallel.ForEachAsync(sessions, async (session, ct) => + { + try + { + await session.CloseAsync(ct).ConfigureAwait(false); + } + catch (ServiceResultException) + { + // the server may already be shutting the session down + } + session.Dispose(); + }).ConfigureAwait(false); + await server.StandardInput.WriteLineAsync("quit").ConfigureAwait(false); + if (!server.WaitForExit(10000)) + { + server.Kill(); + } + return 0; } - string url = ready.Split(' ')[1]; - Console.WriteLine($"server pid={server.Id} url={url}"); - - ITelemetryContext telemetry = CreateTelemetry(LogLevel.Error); - var client = new ClientFixture(telemetry) { SessionTimeout = 60000, OperationTimeout = 60000 }; - await client.LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient").ConfigureAwait(false); - string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; - ArrayOf endpoints = await client.GetEndpointsAsync(new Uri(url)).ConfigureAwait(false); - MessageSecurityMode wantedMode = security switch - { - "sign" => MessageSecurityMode.Sign, - "encrypt" => MessageSecurityMode.SignAndEncrypt, - _ => MessageSecurityMode.None - }; - var filtered = new List(); - foreach (EndpointDescription ep in endpoints) + + private static void PumpServerOutput(Process server, BlockingCollection lines) { - if (ep.SecurityPolicyUri == policy && ep.SecurityMode == wantedMode) + string? line; + while ((line = server.StandardOutput.ReadLine()) != null) { - filtered.Add(ep); + if (line.StartsWith("READY", StringComparison.Ordinal) || + line.StartsWith("REPORT", StringComparison.Ordinal) || + line.StartsWith("MARKED", StringComparison.Ordinal) || + line.StartsWith("ERROR", StringComparison.Ordinal)) + { + lines.Add(line); + } + else if (s_args.ContainsKey("verbose")) + { + Console.WriteLine("[server] " + line); + } } + lines.CompleteAdding(); } - var sessions = new ISession[sessionsCount]; - var connectSw = Stopwatch.StartNew(); - await Parallel.ForAsync(0, sessionsCount, new ParallelOptions { MaxDegreeOfParallelism = 32 }, async (i, ct) => - sessions[i] = await client.ConnectAsync(new Uri(url), policy, filtered).ConfigureAwait(false)).ConfigureAwait(false); - Console.WriteLine($"connected {sessionsCount} sessions ({security}) in {connectSw.ElapsedMilliseconds} ms"); - - Workload workload = scenario switch + private static Process? StartTracer(int processId, int durationSeconds, out string? trace) { - "sub" => new SubscribeWorkload(sessions, ArgInt("nodes", 100), ArgInt("subs", 5), ArgInt("pub", 100), ArgInt("write-interval", 100)), - _ => new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)) - }; - await workload.SetupAsync().ConfigureAwait(false); - - using var stop = new CancellationTokenSource(); - Task run = workload.RunAsync(stop.Token); - await Task.Delay(TimeSpan.FromSeconds(warmupSec)).ConfigureAwait(false); - - Process? tracer = null; - string? trace = s_args.TryGetValue("trace", out string? t) ? t : null; - if (trace != null) - { - string profile = Arg("trace-profile", "cpu"); - string providers = profile switch + if (!s_args.TryGetValue("trace", out trace)) + { + return null; + } + string providers = Arg("trace-profile", "cpu") switch { - // allocation sampling (GCAllocationTick ~ every 100KB) + GC events - "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5 --clrevents gc --clreventlevel verbose", + // allocation sampling (GCAllocationTick, about every 100 KB) and GC events + "alloc" => "--providers Microsoft-Windows-DotNETRuntime:0x1:5", "gc" => "--profile gc-verbose", + // the Contention keyword "contention" => "--providers Microsoft-Windows-DotNETRuntime:0x4000:4", _ => "--profile dotnet-sampled-thread-time" }; - tracer = Process.Start(new ProcessStartInfo("dotnet-trace", $"collect --process-id {server.Id} {providers} -o \"{trace}\" --duration 00:00:{durationSec:D2}") + return Process.Start(new ProcessStartInfo( + "dotnet-trace", + FormattableString.Invariant( + $"collect --process-id {processId} {providers} -o \"{trace}\" --duration 00:00:{durationSeconds:D2}")) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }); - await Task.Delay(2000).ConfigureAwait(false); } - server.StandardInput.WriteLine("mark"); - Take(serverLines, "MARKED"); - workload.ResetStats(); - var sw = Stopwatch.StartNew(); - await Task.Delay(TimeSpan.FromSeconds(durationSec)).ConfigureAwait(false); - server.StandardInput.WriteLine("report"); - string report = Take(serverLines, "REPORT"); - double elapsed = sw.Elapsed.TotalSeconds; - string clientReport = workload.Report(elapsed); - stop.Cancel(); - try + private static string Take(BlockingCollection lines, string prefix) { - await run.ConfigureAwait(false); + while (true) + { + string line = lines.Take(); + if (line.StartsWith(prefix, StringComparison.Ordinal)) + { + return line; + } + } } - catch (OperationCanceledException) + + private static (double AllocatedMb, double CpuMs) ParseServerReport(string report) { + Dictionary values = report + .Split(' ') + .Select(p => p.Split('=')) + .Where(p => p.Length == 2) + .ToDictionary(p => p[0], p => p[1], StringComparer.Ordinal); + return ( + double.Parse(values["allocMB"], CultureInfo.InvariantCulture), + double.Parse(values["cpuMs"], CultureInfo.InvariantCulture)); } + } - if (tracer != null) + /// + /// Process-wide counters of the server process, captured at the start and end of the window. + /// + internal sealed record ServerStats( + long Allocated, + int Gen0, + int Gen1, + int Gen2, + TimeSpan Pause, + TimeSpan Cpu, + long Contention, + long Timestamp, + int Threads, + long WorkingSet, + long Heap) + { + public static ServerStats Capture() { - await tracer.WaitForExitAsync().ConfigureAwait(false); - Console.WriteLine("trace: " + trace); + using var process = Process.GetCurrentProcess(); + return new ServerStats( + GC.GetTotalAllocatedBytes(true), + GC.CollectionCount(0), + GC.CollectionCount(1), + GC.CollectionCount(2), + GC.GetTotalPauseDuration(), + process.TotalProcessorTime, + Monitor.LockContentionCount, + Stopwatch.GetTimestamp(), + ThreadPool.ThreadCount, + process.WorkingSet64, + GC.GetGCMemoryInfo().HeapSizeBytes); + } + + public string Delta(ServerStats mark) + { + double seconds = Stopwatch.GetElapsedTime(mark.Timestamp, Timestamp).TotalSeconds; + TimeSpan cpu = Cpu - mark.Cpu; + return FormattableString.Invariant( + $"sec={seconds:F2} allocMB={(Allocated - mark.Allocated) / 1048576.0:F1} gen0={Gen0 - mark.Gen0} ") + + FormattableString.Invariant( + $"gen1={Gen1 - mark.Gen1} gen2={Gen2 - mark.Gen2} pauseMs={(Pause - mark.Pause).TotalMilliseconds:F0} ") + + FormattableString.Invariant( + $"cpuMs={cpu.TotalMilliseconds:F0} cpuCores={cpu.TotalSeconds / seconds:F2} ") + + FormattableString.Invariant( + $"contention={Contention - mark.Contention} threads={Threads} wsMB={WorkingSet / 1048576} heapMB={Heap / 1048576}"); } + } - var sb = new StringBuilder(); - sb.Append(CultureInfo.InvariantCulture, $"RESULT scenario={scenario} security={security} sessions={sessionsCount} {string.Join(' ', args.Skip(1))}\n"); - sb.Append(" client: ").Append(clientReport).Append('\n'); - sb.Append(" server: ").Append(report[7..]).Append('\n'); - double ops = workload.Operations; - if (ops > 0) + /// + /// Latency samples (Stopwatch ticks) recorded by many threads. ConcurrentBag keeps a + /// list per thread, so recording does not contend between the load threads. + /// + internal sealed class LatencyRecorder + { + public void Record(long ticks) { - ServerReport sr = ServerReport.Parse(report[7..]); - sb.Append(CultureInfo.InvariantCulture, $" per-op: allocB={sr.AllocMb * 1048576 / ops:F0} cpuUs={sr.CpuMs * 1000 / ops:F1} ({workload.OperationName})\n"); + m_samples.Add(ticks); } - Console.Write(sb.ToString()); - if (s_args.TryGetValue("out", out string? outFile)) + + public void Clear() { - File.AppendAllText(outFile, sb.ToString()); + m_samples.Clear(); } - await Parallel.ForEachAsync(sessions, async (s, ct) => + public string Percentiles() { - try + var all = new List(m_samples); + if (all.Count == 0) { - await s.CloseAsync(ct).ConfigureAwait(false); + return "n=0"; } - catch + all.Sort(); + double Ms(double quantile) { + return all[Math.Min(all.Count - 1, (int)(quantile * all.Count))] * 1000.0 / Stopwatch.Frequency; } - s.Dispose(); - }).ConfigureAwait(false); - server.StandardInput.WriteLine("quit"); - if (!server.WaitForExit(10000)) - { - server.Kill(); + return FormattableString.Invariant( + $"n={all.Count} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms ") + + FormattableString.Invariant($"p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); } - return 0; + + private readonly ConcurrentBag m_samples = []; } - private static string Take(BlockingCollection lines, string prefix) + /// + /// A load pattern driven against the server. + /// + internal abstract class Workload { - while (true) - { - string l = lines.Take(); - if (l.StartsWith(prefix, StringComparison.Ordinal)) - { - return l; - } - } + public abstract double Operations { get; } + + public abstract string OperationName { get; } + + public abstract Task SetupAsync(); + + public abstract Task RunAsync(CancellationToken ct); + + public abstract void ResetStats(); + + public abstract string Report(double seconds); } - private sealed record ServerReport(double AllocMb, double CpuMs) + /// + /// Closed-loop Read, Write or Browse requests from every session. + /// + internal sealed class RequestWorkload : Workload { - public static ServerReport Parse(string s) + public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) { - var d = s.Split(' ').Select(p => p.Split('=')).Where(p => p.Length == 2).ToDictionary(p => p[0], p => p[1]); - return new ServerReport(double.Parse(d["allocMB"], CultureInfo.InvariantCulture), double.Parse(d["cpuMs"], CultureInfo.InvariantCulture)); + m_kind = kind; + m_sessions = sessions; + m_nodes = nodes; + m_inflight = inflight; } - } - internal static List GetNodes(ISession session, int count, Type? onlyType = null) - { - var result = new List(); - foreach ((Type type, ExpandedNodeId[] ids) in CommonTestWorkers.NodeIdTestSetStaticMassNumeric) + public override double Operations => Interlocked.Read(ref m_requests); + + public override string OperationName => "request"; + + public override Task SetupAsync() { - if (onlyType != null && type != onlyType) + List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); + m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); + m_write = nodes.Select(n => new WriteValue { - continue; - } - foreach (ExpandedNodeId e in ids) + NodeId = n, + AttributeId = Attributes.Value, + Value = new DataValue(new Variant(42)) + }).ToArray(); + m_browse = new[] { - if (result.Count == count) + new BrowseDescription { - return result; + NodeId = ObjectIds.Server, + BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, + IncludeSubtypes = true, + ResultMask = (uint)BrowseResultMask.All } - result.Add(ExpandedNodeId.ToNodeId(e, session.NamespaceUris)); - } + }; + Console.WriteLine(FormattableString.Invariant( + $"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session")); + return Task.CompletedTask; } - return result; - } -} -internal abstract class Workload -{ - public abstract Task SetupAsync(); - public abstract Task RunAsync(CancellationToken ct); - public abstract void ResetStats(); - public abstract string Report(double seconds); - public abstract double Operations { get; } - public abstract string OperationName { get; } - - protected static string Percentiles(List ticksList) - { - if (ticksList.Count == 0) + public override async Task RunAsync(CancellationToken ct) { - return "n=0"; + var tasks = new List(); + foreach (ISession session in m_sessions) + { + for (int i = 0; i < m_inflight; i++) + { + tasks.Add(Task.Run(() => LoopAsync(session, ct), CancellationToken.None)); + } + } + await Task.WhenAll(tasks).ConfigureAwait(false); } - long[] a = [.. ticksList]; - Array.Sort(a); - double Ms(double q) => a[Math.Min(a.Length - 1, (int)(q * a.Length))] * 1000.0 / Stopwatch.Frequency; - return string.Create(CultureInfo.InvariantCulture, - $"n={a.Length} p50={Ms(0.5):F3}ms p90={Ms(0.9):F3}ms p99={Ms(0.99):F3}ms p999={Ms(0.999):F3}ms max={Ms(1):F3}ms"); - } -} -internal sealed class RequestWorkload : Workload -{ - private readonly string m_kind; - private readonly ISession[] m_sessions; - private readonly int m_nodes; - private readonly int m_inflight; - private readonly ConcurrentBag> m_lat = new(); - private readonly ThreadLocal> m_local; - private volatile int m_epoch; - private long m_requests; - private long m_errors; - private ArrayOf m_read; - private ArrayOf m_write; - private ArrayOf m_browse; - - public RequestWorkload(string kind, ISession[] sessions, int nodes, int inflight) - { - m_kind = kind; - m_sessions = sessions; - m_nodes = nodes; - m_inflight = inflight; - m_local = new ThreadLocal>(() => + public override void ResetStats() { - var l = new List(1 << 16); - m_lat.Add(l); - return l; - }, false); - } - - public override double Operations => Interlocked.Read(ref m_requests); - public override string OperationName => "request"; - - public override Task SetupAsync() - { - List nodes = Program.GetNodes(m_sessions[0], m_nodes, m_kind == "write" ? typeof(int) : null); - m_read = nodes.Select(n => new ReadValueId { NodeId = n, AttributeId = Attributes.Value }).ToArray(); - m_write = nodes.Select(n => new WriteValue { NodeId = n, AttributeId = Attributes.Value, Value = new DataValue(new Variant(42)) }).ToArray(); - m_browse = new[] { new BrowseDescription { NodeId = Opc.Ua.ObjectIds.Server, BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All } }; - Console.WriteLine($"{m_kind}: {nodes.Count} nodes/request, inflight {m_inflight}/session"); - return Task.CompletedTask; - } + Interlocked.Increment(ref m_epoch); + m_latency.Clear(); + Interlocked.Exchange(ref m_requests, 0); + Interlocked.Exchange(ref m_errors, 0); + } - public override async Task RunAsync(CancellationToken ct) - { - var tasks = new List(); - foreach (ISession s in m_sessions) + public override string Report(double seconds) { - for (int i = 0; i < m_inflight; i++) - { - tasks.Add(Task.Run(() => LoopAsync(s, ct), CancellationToken.None)); - } + long requests = Interlocked.Read(ref m_requests); + int valuesPerRequest = m_kind == "browse" ? 1 : m_nodes; + return FormattableString.Invariant( + $"req/s={requests / seconds:F0} values/s={requests * valuesPerRequest / seconds:F0} ") + + FormattableString.Invariant($"errors={Interlocked.Read(ref m_errors)} latency {m_latency.Percentiles()}"); } - await Task.WhenAll(tasks).ConfigureAwait(false); - } - private async Task LoopAsync(ISession s, CancellationToken ct) - { - while (!ct.IsCancellationRequested) + private async Task LoopAsync(ISession session, CancellationToken ct) { - long start = Stopwatch.GetTimestamp(); - int epoch = m_epoch; - try + while (!ct.IsCancellationRequested) { - switch (m_kind) + long start = Stopwatch.GetTimestamp(); + int epoch = Volatile.Read(ref m_epoch); + try { - case "write": - await s.WriteAsync(null, m_write, ct).ConfigureAwait(false); - break; - case "browse": - await s.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); - break; - default: - await s.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); - break; + switch (m_kind) + { + case "write": + await session.WriteAsync(null, m_write, ct).ConfigureAwait(false); + break; + case "browse": + await session.BrowseAsync(null, null, 0, m_browse, ct).ConfigureAwait(false); + break; + default: + await session.ReadAsync(null, 0, TimestampsToReturn.Both, m_read, ct).ConfigureAwait(false); + break; + } } - } - catch (OperationCanceledException) - { - return; - } - catch (Exception ex) - { - if (Interlocked.Increment(ref m_errors) < 5) + catch (OperationCanceledException) { - Console.WriteLine("error: " + ex.Message); + return; } - continue; - } - long el = Stopwatch.GetTimestamp() - start; - if (epoch == m_epoch) - { - List l = m_local.Value!; - lock (l) + catch (ServiceResultException ex) { - l.Add(el); + if (Interlocked.Increment(ref m_errors) < 5) + { + Console.WriteLine("error: " + ex.Message); + } + continue; + } + if (epoch == Volatile.Read(ref m_epoch)) + { + m_latency.Record(Stopwatch.GetTimestamp() - start); + Interlocked.Increment(ref m_requests); } - Interlocked.Increment(ref m_requests); } } + + private readonly string m_kind; + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_inflight; + private readonly LatencyRecorder m_latency = new(); + private int m_epoch; + private long m_requests; + private long m_errors; + private ArrayOf m_read; + private ArrayOf m_write; + private ArrayOf m_browse; } - public override void ResetStats() + /// + /// Data-change subscriptions on every session and one writer changing the monitored values. + /// The written value is a counter, so each notification maps back to the time of its write. + /// + internal sealed class SubscribeWorkload : Workload { - m_epoch++; - foreach (List l in m_lat) + public SubscribeWorkload(ISession[] sessions, int nodes, int subscriptions, int publishingInterval, int writeInterval) { - lock (l) - { - l.Clear(); - } + m_sessions = sessions; + m_nodes = nodes; + m_subscriptions = subscriptions; + m_publishingInterval = publishingInterval; + m_writeInterval = writeInterval; } - Interlocked.Exchange(ref m_requests, 0); - Interlocked.Exchange(ref m_errors, 0); - } - public override string Report(double seconds) - { - var all = new List(); - foreach (List l in m_lat) + public override double Operations => Interlocked.Read(ref m_notifications); + + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); + var setupTime = Stopwatch.StartNew(); + await Parallel.ForEachAsync( + m_sessions, + new ParallelOptions { MaxDegreeOfParallelism = 16 }, + async (session, ct) => + { + for (int i = 0; i < m_subscriptions; i++) + { + await CreateSubscriptionAsync(session, ct).ConfigureAwait(false); + } + }).ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"sub: {m_sessions.Length} sessions x {m_subscriptions} subs x {m_nodeIds.Count} items, ") + + FormattableString.Invariant( + $"pub {m_publishingInterval} ms, write every {m_writeInterval} ms; setup {setupTime.ElapsedMilliseconds} ms")); + } + + public override async Task RunAsync(CancellationToken ct) { - lock (l) + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); + while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) { - all.AddRange(l); + int counter = Interlocked.Increment(ref m_counter); + var values = new WriteValue[m_nodeIds.Count]; + for (int i = 0; i < values.Length; i++) + { + values[i] = new WriteValue + { + NodeId = m_nodeIds[i], + AttributeId = Attributes.Value, + Value = new DataValue(new Variant(counter)) + }; + } + Volatile.Write(ref m_writeTicks[counter & 0xFFFF], Stopwatch.GetTimestamp()); + await m_sessions[0].WriteAsync(null, values, ct).ConfigureAwait(false); + Interlocked.Increment(ref m_writes); } } - long req = Interlocked.Read(ref m_requests); - return string.Create(CultureInfo.InvariantCulture, - $"req/s={req / seconds:F0} values/s={req * (m_kind == "browse" ? 1 : m_nodes) / seconds:F0} errors={m_errors} latency {Percentiles(all)}"); - } -} -internal sealed class SubscribeWorkload : Workload -{ - private readonly ISession[] m_sessions; - private readonly int m_nodes; - private readonly int m_subs; - private readonly int m_pub; - private readonly int m_writeInterval; - private readonly long[] m_writeTicks = new long[1 << 16]; - private readonly ConcurrentBag> m_lat = new(); - private readonly ThreadLocal> m_local; - private long m_notifications; - private long m_writes; - private int m_counter; - private ISession? m_writer; - private ArrayOf m_nodeIds; - - public SubscribeWorkload(ISession[] sessions, int nodes, int subs, int pub, int writeInterval) - { - m_sessions = sessions; - m_nodes = nodes; - m_subs = subs; - m_pub = pub; - m_writeInterval = writeInterval; - m_local = new ThreadLocal>(() => + public override void ResetStats() { - var l = new List(1 << 16); - m_lat.Add(l); - return l; - }, false); - } + m_latency.Clear(); + Interlocked.Exchange(ref m_notifications, 0); + Interlocked.Exchange(ref m_writes, 0); + } - public override double Operations => Interlocked.Read(ref m_notifications); - public override string OperationName => "notification"; + public override string Report(double seconds) + { + long notifications = Interlocked.Read(ref m_notifications); + long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subscriptions * m_nodeIds.Count; + double ratio = expected == 0 ? 0 : (double)notifications / expected; + return FormattableString.Invariant( + $"notif/s={notifications / seconds:F0} received/expected={ratio:P1} ") + + FormattableString.Invariant($"write->notify latency {m_latency.Percentiles()}"); + } - public override async Task SetupAsync() - { - m_nodeIds = Program.GetNodes(m_sessions[0], m_nodes, typeof(int)).ToArray(); - m_writer = m_sessions[0]; - var sw = Stopwatch.StartNew(); - await Parallel.ForEachAsync(m_sessions, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (session, ct) => + private async Task CreateSubscriptionAsync(ISession session, CancellationToken ct) { - for (int j = 0; j < m_subs; j++) +#pragma warning disable CA2000 // ownership transfers to the session in AddSubscription; disposed below on failure before that + var subscription = new Subscription(session.DefaultSubscription) { - var subscription = new Subscription(session.DefaultSubscription) - { - PublishingInterval = m_pub, - KeepAliveCount = 10, - LifetimeCount = 100 - }; - foreach (NodeId n in m_nodeIds) + PublishingInterval = m_publishingInterval, + KeepAliveCount = 10, + LifetimeCount = 100, + FastDataChangeCallback = OnDataChange + }; +#pragma warning restore CA2000 + try + { + foreach (NodeId nodeId in m_nodeIds) { subscription.AddItem(new MonitoredItem(subscription.DefaultItem) { - StartNodeId = n, + StartNodeId = nodeId, AttributeId = Attributes.Value, MonitoringMode = MonitoringMode.Reporting, SamplingInterval = 0, QueueSize = 1 }); } - subscription.FastDataChangeCallback = (sub, notification, _) => - { - long now = Stopwatch.GetTimestamp(); - List l = m_local.Value!; - int count = 0; - lock (l) - { - foreach (MonitoredItemNotification item in notification.MonitoredItems) - { - if (item.Value.WrappedValue.TryGetValue(out int v) && v > 0) - { - long w = Volatile.Read(ref m_writeTicks[v & 0xFFFF]); - if (w != 0) - { - l.Add(now - w); - } - } - count++; - } - } - Interlocked.Add(ref m_notifications, count); - }; + // the session owns and disposes the subscription from here on session.AddSubscription(subscription); - await subscription.CreateAsync(ct).ConfigureAwait(false); } - }).ConfigureAwait(false); - Console.WriteLine($"sub: {m_sessions.Length} sessions x {m_subs} subs x {m_nodeIds.Count} items = {m_sessions.Length * m_subs * m_nodeIds.Count} items, pub {m_pub} ms, write every {m_writeInterval} ms; setup {sw.ElapsedMilliseconds} ms"); - } - - public override async Task RunAsync(CancellationToken ct) - { - using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(m_writeInterval)); - while (await timer.WaitForNextTickAsync(ct).ConfigureAwait(false)) - { - int v = Interlocked.Increment(ref m_counter); - var values = new WriteValue[m_nodeIds.Count]; - for (int i = 0; i < values.Length; i++) + catch { - values[i] = new WriteValue { NodeId = m_nodeIds[i], AttributeId = Attributes.Value, Value = new DataValue(new Variant(v)) }; + subscription.Dispose(); + throw; } - Volatile.Write(ref m_writeTicks[v & 0xFFFF], Stopwatch.GetTimestamp()); - await m_writer!.WriteAsync(null, values, ct).ConfigureAwait(false); - Interlocked.Increment(ref m_writes); + await subscription.CreateAsync(ct).ConfigureAwait(false); } - } - public override void ResetStats() - { - foreach (List l in m_lat) + private void OnDataChange(Subscription subscription, DataChangeNotification notification, ArrayOf stringTable) { - lock (l) + long now = Stopwatch.GetTimestamp(); + int count = 0; + foreach (MonitoredItemNotification item in notification.MonitoredItems) { - l.Clear(); + if (item.Value.WrappedValue.TryGetValue(out int counter) && counter > 0) + { + long written = Volatile.Read(ref m_writeTicks[counter & 0xFFFF]); + if (written != 0) + { + m_latency.Record(now - written); + } + } + count++; } + Interlocked.Add(ref m_notifications, count); } - Interlocked.Exchange(ref m_notifications, 0); - Interlocked.Exchange(ref m_writes, 0); - } - public override string Report(double seconds) - { - var all = new List(); - foreach (List l in m_lat) - { - lock (l) - { - all.AddRange(l); - } - } - long n = Interlocked.Read(ref m_notifications); - long expected = Interlocked.Read(ref m_writes) * m_sessions.Length * m_subs * m_nodeIds.Count; - return string.Create(CultureInfo.InvariantCulture, - $"notif/s={n / seconds:F0} received/expected={(expected == 0 ? 0 : (double)n / expected):P1} write->notify latency {Percentiles(all)}"); + private readonly ISession[] m_sessions; + private readonly int m_nodes; + private readonly int m_subscriptions; + private readonly int m_publishingInterval; + private readonly int m_writeInterval; + private readonly long[] m_writeTicks = new long[1 << 16]; + private readonly LatencyRecorder m_latency = new(); + private long m_notifications; + private long m_writes; + private int m_counter; + private ArrayOf m_nodeIds; } } diff --git a/perf/ServerLoadHarness/ServerLoadHarness.csproj b/perf/ServerLoadHarness/ServerLoadHarness.csproj index 1d238e7b18..6fae6cf2bb 100644 --- a/perf/ServerLoadHarness/ServerLoadHarness.csproj +++ b/perf/ServerLoadHarness/ServerLoadHarness.csproj @@ -1,17 +1,14 @@ + Exe net10.0 - enable - enable false + false true - true - $(NoWarn);CS1591;CA1303;CA2007;CA1031;CA2000;CA1849;CA5394;RCS1090;CA1515;CA1305;CA1307;CA1310;IDE0005;CA1062;CA1822;CA1859;CA1861;CS0618;RCS1163;CA1002;CA1812;CA1852;CA1050;CA2201;CA1001 - false - false - none diff --git a/perf/TraceAnalyzer/Program.cs b/perf/TraceAnalyzer/Program.cs index a8002ec6eb..175d9de63d 100644 --- a/perf/TraceAnalyzer/Program.cs +++ b/perf/TraceAnalyzer/Program.cs @@ -1,174 +1,276 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + // Trace analyzer for perf/ServerLoadHarness traces. Not part of the product; see perf/README.md. -// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped), exclusive/inclusive, callers of focus -// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, by first Opc.Ua frame, and stacks -// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack +// TraceAnalyzer cpu [top] [focus] : on-CPU samples (blocked leaf frames dropped) +// TraceAnalyzer alloc [top] [type] : GCAllocationTick bytes by type, first Opc.Ua frame, stacks +// TraceAnalyzer contention [top] : ContentionStart events by Opc.Ua stack + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Etlx; using Microsoft.Diagnostics.Tracing.Parsers.Clr; -string mode = args[0]; -string file = args[1]; -int top = args.Length > 2 ? int.Parse(args[2]) : 40; -string focus = args.Length > 3 ? args[3] : null; -string[] s_blocked = { "LowLevelLifoSemaphore.Wait", "WaitForSignal", "WorkerThreadStart", "IOCompletionPoller.Poll", "WaitOneNoCheck", "Thread.Sleep", "PollGCWorker|70_0", "WaitMultipleIgnoringSyncContext", "Monitor.Wait", "!?" }; - -string etlx = TraceLog.CreateFromEventPipeDataFile(file, null, new TraceLogOptions { ContinueOnError = true }); -using var log = new TraceLog(etlx); - -static string Name(TraceCallStack s) +namespace Opc.Ua.Perf.TraceAnalyzer { - var m = s.CodeAddress.FullMethodName; - if (string.IsNullOrEmpty(m)) + /// + /// Aggregates EventPipe traces recorded by the server load harness. + /// + public static class Program { - m = s.CodeAddress.ModuleName + "!?"; - } - return m; -} + /// + /// Leaf frames of threads that wait rather than run. The sampled thread-time + /// profile samples blocked threads too, so these are dropped to approximate CPU. + /// + private static readonly string[] s_blockedLeafFrames = + [ + "LowLevelLifoSemaphore.Wait", + "WaitForSignal", + "WorkerThreadStart", + "IOCompletionPoller.Poll", + "WaitOneNoCheck", + "Thread.Sleep", + "PollGCWorker|70_0", + "WaitMultipleIgnoringSyncContext", + "Monitor.Wait", + "!?" + ]; -static string Short(string m) -{ - int p = m.IndexOf('('); - return p > 0 ? m[..p] : m; -} + /// + /// Entry point. + /// + public static int Main(string[] args) + { + if (args.Length < 2) + { + Console.WriteLine("usage: TraceAnalyzer cpu|alloc|contention [top] [focus]"); + return 1; + } -static bool IsOurs(string m) => m.StartsWith("Opc.Ua", StringComparison.Ordinal) || m.StartsWith("Quickstarts", StringComparison.Ordinal); + string mode = args[0]; + int top = args.Length > 2 ? int.Parse(args[2], CultureInfo.InvariantCulture) : 40; + string? focus = args.Length > 3 ? args[3] : null; -if (mode == "contention") -{ - var byStack = new Dictionary(); - int total = 0; - foreach (TraceEvent e in log.Events) - { - if (e.EventName == null || !e.EventName.StartsWith("Contention/Start")) continue; - total++; - var chain = new List(); - for (TraceCallStack f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) { string n = Short(Name(f)); if (IsOurs(n) || chain.Count == 0) chain.Add(n); } - string k = string.Join("\n ", chain); - byStack[k] = byStack.GetValueOrDefault(k) + 1; - } - Console.WriteLine($"contention events: {total}"); - foreach (var kv in byStack.OrderByDescending(k => k.Value).Take(top)) Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); - return; -} -if (mode == "cpu") -{ - var excl = new Dictionary(); - var incl = new Dictionary(); - var firstOurs = new Dictionary(); - var callers = new Dictionary(); - int total = 0, all = 0; - foreach (TraceEvent e in log.Events) - { - if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") - { - continue; - } - all++; - object type = e.PayloadByName("Type"); - if (type != null && Convert.ToInt32(type) != 2) - { - continue; // not running managed code - } - TraceCallStack s = e.CallStack(); - if (s == null) - { - continue; + string etlx = TraceLog.CreateFromEventPipeDataFile( + args[1], null, new TraceLogOptions { ContinueOnError = true }); + using var log = new TraceLog(etlx); + + switch (mode) + { + case "contention": + Contention(log, top); + break; + case "cpu": + Cpu(log, top, focus); + break; + default: + Allocations(log, top, focus); + break; + } + return 0; } - string leaf = Short(Name(s)); - if (s_blocked.Any(b => leaf.EndsWith(b))) continue; - total++; - excl[leaf] = excl.GetValueOrDefault(leaf) + 1; - var seen = new HashSet(); - bool gotOurs = false; - bool focusHit = false; - for (TraceCallStack f = s; f != null; f = f.Caller) + + private static void Contention(TraceLog log, int top) { - string n = Short(Name(f)); - if (seen.Add(n)) + var byStack = new Dictionary(StringComparer.Ordinal); + double total = 0; + foreach (TraceEvent e in log.Events) { - incl[n] = incl.GetValueOrDefault(n) + 1; + if (e.EventName?.StartsWith("Contention/Start", StringComparison.Ordinal) != true) + { + continue; + } + total++; + var chain = new List(); + for (TraceCallStack? f = e.CallStack(); f != null && chain.Count < 7; f = f.Caller) + { + string n = Short(Name(f)); + if (IsOurs(n) || chain.Count == 0) + { + chain.Add(n); + } + } + Add(byStack, string.Join("\n ", chain), 1); } - if (!gotOurs && IsOurs(n)) + Console.WriteLine(FormattableString.Invariant($"contention events: {total}")); + Dump("stacks", byStack, total, top); + } + + private static void Cpu(TraceLog log, int top, string? focus) + { + var exclusive = new Dictionary(StringComparer.Ordinal); + var inclusive = new Dictionary(StringComparer.Ordinal); + var firstOurs = new Dictionary(StringComparer.Ordinal); + var callers = new Dictionary(StringComparer.Ordinal); + double total = 0; + int all = 0; + foreach (TraceEvent e in log.Events) { - gotOurs = true; - firstOurs[n] = firstOurs.GetValueOrDefault(n) + 1; + if (e.ProviderName != "Microsoft-DotNETCore-SampleProfiler") + { + continue; + } + all++; + object? type = e.PayloadByName("Type"); + if (type != null && Convert.ToInt32(type, CultureInfo.InvariantCulture) != 2) + { + // not running managed code + continue; + } + TraceCallStack? stack = e.CallStack(); + if (stack == null) + { + continue; + } + string leaf = Short(Name(stack)); + if (s_blockedLeafFrames.Any(b => leaf.EndsWith(b, StringComparison.Ordinal))) + { + continue; + } + total++; + Add(exclusive, leaf, 1); + var seen = new HashSet(StringComparer.Ordinal); + bool gotOurs = false; + bool focusHit = false; + for (TraceCallStack? f = stack; f != null; f = f.Caller) + { + string n = Short(Name(f)); + if (seen.Add(n)) + { + Add(inclusive, n, 1); + } + if (!gotOurs && IsOurs(n)) + { + gotOurs = true; + Add(firstOurs, n, 1); + } + if (focus != null && !focusHit && n.Contains(focus, StringComparison.Ordinal)) + { + focusHit = true; + Add(callers, f.Caller != null ? Short(Name(f.Caller)) : "", 1); + } + } } - if (focus != null && !focusHit && n.Contains(focus)) + Console.WriteLine(FormattableString.Invariant($"samples: on-cpu={total} all={all}")); + Dump("exclusive", exclusive, total, top); + Dump("first Opc.Ua frame", firstOurs, total, top); + Dump( + "inclusive (Opc.Ua only)", + inclusive.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value, StringComparer.Ordinal), + total, + top); + if (focus != null) { - focusHit = true; - string c = f.Caller != null ? Short(Name(f.Caller)) : ""; - callers[c] = callers.GetValueOrDefault(c) + 1; + Dump("callers of " + focus, callers, total, top); } } - } - Console.WriteLine($"samples: managed-running={total} all={all}"); - void Dump(string title, Dictionary d, int n) - { - Console.WriteLine($"--- {title}"); - foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) - { - Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); - } - } - Dump("exclusive", excl, top); - Dump("first Opc.Ua frame (exclusive-ish)", firstOurs, top); - Dump("inclusive (Opc.Ua only)", incl.Where(k => IsOurs(k.Key)).ToDictionary(k => k.Key, k => k.Value), top); - if (focus != null) - { - Dump("callers of " + focus, callers, top); - } -} -else -{ - var byType = new Dictionary(); - var byFrame = new Dictionary(); - var byStack = new Dictionary(); - double total = 0; - foreach (TraceEvent e in log.Events) - { - if (e is not GCAllocationTickTraceData a) - { - continue; - } - double amount = a.AllocationAmount64; - total += amount; - string t = a.TypeName ?? "?"; - byType[t] = byType.GetValueOrDefault(t) + amount; - TraceCallStack s = e.CallStack(); - string frame = "?"; - var chain = new List(); - for (TraceCallStack f = s; f != null; f = f.Caller) + + private static void Allocations(TraceLog log, int top, string? focus) { - string n = Short(Name(f)); - if (IsOurs(n)) + var byType = new Dictionary(StringComparer.Ordinal); + var byFrame = new Dictionary(StringComparer.Ordinal); + var byStack = new Dictionary(StringComparer.Ordinal); + double total = 0; + foreach (TraceEvent e in log.Events) { - if (frame == "?") + if (e is not GCAllocationTickTraceData tick) + { + continue; + } + double amount = tick.AllocationAmount64; + total += amount; + string type = tick.TypeName ?? "?"; + Add(byType, type, amount); + string frame = "?"; + var chain = new List(); + for (TraceCallStack? f = e.CallStack(); f != null; f = f.Caller) { - frame = n; + string n = Short(Name(f)); + if (!IsOurs(n)) + { + continue; + } + if (frame == "?") + { + frame = n; + } + if (chain.Count < 6 || (focus != null && chain.Count < 12)) + { + chain.Add(n); + } } - if (chain.Count < 6) + Add(byFrame, type + " <- " + frame, amount); + if (focus == null || type.Contains(focus, StringComparison.Ordinal)) { - chain.Add(n); + Add(byStack, type + "\n " + string.Join("\n ", chain), amount); } } + Console.WriteLine(FormattableString.Invariant($"sampled alloc total: {total / 1048576:F1} MB")); + Dump("by type", byType, total, top); + Dump("by type <- first Opc.Ua frame", byFrame, total, top); + Dump("stacks", byStack, total, top / 2); } - byFrame[t + " <- " + frame] = byFrame.GetValueOrDefault(t + " <- " + frame) + amount; - if (focus == null || t.Contains(focus)) + + private static void Add(Dictionary totals, string key, double amount) { - string k = t + "\n " + string.Join("\n ", chain); - byStack[k] = byStack.GetValueOrDefault(k) + amount; + totals[key] = totals.GetValueOrDefault(key) + amount; } - } - Console.WriteLine($"sampled alloc total: {total / 1048576:F1} MB"); - void Dump(string title, Dictionary d, int n) - { - Console.WriteLine($"--- {title}"); - foreach (var kv in d.OrderByDescending(k => k.Value).Take(n)) + + private static void Dump(string title, Dictionary totals, double total, int count) + { + Console.WriteLine("--- " + title); + foreach (KeyValuePair kv in totals.OrderByDescending(k => k.Value).Take(count)) + { + Console.WriteLine(FormattableString.Invariant($"{100.0 * kv.Value / total,6:F2}% {kv.Key}")); + } + } + + private static string Name(TraceCallStack frame) + { + string name = frame.CodeAddress.FullMethodName; + return string.IsNullOrEmpty(name) ? frame.CodeAddress.ModuleName + "!?" : name; + } + + private static string Short(string method) + { + int p = method.IndexOf('(', StringComparison.Ordinal); + return p > 0 ? method[..p] : method; + } + + private static bool IsOurs(string method) { - Console.WriteLine($"{100.0 * kv.Value / total,6:F2}% {kv.Key}"); + return method.StartsWith("Opc.Ua", StringComparison.Ordinal) || + method.StartsWith("Quickstarts", StringComparison.Ordinal); } } - Dump("by type", byType, top); - Dump("by type <- first Opc.Ua frame", byFrame, top); - Dump("stacks", byStack, top / 2); } diff --git a/perf/TraceAnalyzer/TraceAnalyzer.csproj b/perf/TraceAnalyzer/TraceAnalyzer.csproj index e2529b9c27..b43a522d76 100644 --- a/perf/TraceAnalyzer/TraceAnalyzer.csproj +++ b/perf/TraceAnalyzer/TraceAnalyzer.csproj @@ -3,13 +3,7 @@ Exe net10.0 - disable - enable false - false - false - none - $(NoWarn);CS1591;CA1305;CA1307;CA1310;CA1849;RCS1007;IDE0011 From dc638ab9fae778e6fb3c10c4eb6632dfb91594f1 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 08:21:43 +0200 Subject: [PATCH 07/19] Fix certificate checks after the ByteString migration; cover queue and pool edge cases - StandardServer compared the session certificate with the channel certificate through Utils.IsEqual, which bound to the object overload once the channel certificate became a ByteString and always failed (BadSecurityChecksFailed on secure CreateSession and TransferSubscriptions). Compare the spans. - The isolation provider's ChannelSnapshot copies the certificate again: the context wraps a buffer the transport owns, and revalidation must detect evidence that changed after the owner was issued. - PeerMatches has the array fallback only on .NET Framework; on .NET the 16-byte stack buffer always holds the address. - Tests for a stopped FairRequestQueue (admission and readers) and for the ObjectPool capacity argument. Co-Authored-By: Claude Opus 5.5 --- .../DefaultServerResourceIsolationProvider.cs | 15 +++++----- src/Opc.Ua.Server/Server/StandardServer.cs | 5 ++-- .../Stack/Server/FairRequestQueueTests.cs | 30 +++++++++++++++++++ .../Types/Utils/ObjectPoolTests.cs | 9 ++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs index 85fb0babca..60d2b7deba 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/DefaultServerResourceIsolationProvider.cs @@ -844,9 +844,10 @@ public ChannelSnapshot(SecureChannelContext context) m_channelId = context.SecureChannelId; SecurityPolicyUri = context.EndpointDescription?.SecurityPolicyUri; SecurityMode = context.EndpointDescription?.SecurityMode ?? MessageSecurityMode.Invalid; - // ByteString is a read-only view, so the evidence cannot change after capture. + // Copied: the context wraps a buffer the transport owns, and revalidation + // must detect evidence that changed after the owner was issued. m_certificate = SecurityPolicyUri != SecurityPolicies.None - ? context.ClientChannelCertificate : default; + ? ByteString.From(context.ClientChannelCertificate.Span) : default; m_peer = context.PeerAddress?.GetAddressBytes(); m_upstreamIdentity = context.UpstreamIdentity; } @@ -886,13 +887,13 @@ private bool PeerMatches(IPAddress? peer) return m_peer == null || m_peer.Length == 0; } #if NET + // 16 bytes hold an IPv6 address, so the write always succeeds. Span bytes = stackalloc byte[16]; - if (peer.TryWriteBytes(bytes, out int written)) - { - return m_peer.AsSpan().SequenceEqual(bytes[..written]); - } -#endif + return peer.TryWriteBytes(bytes, out int written) && + m_peer.AsSpan().SequenceEqual(bytes[..written]); +#else return m_peer.AsSpan().SequenceEqual(peer.GetAddressBytes()); +#endif } private readonly string m_channelId; diff --git a/src/Opc.Ua.Server/Server/StandardServer.cs b/src/Opc.Ua.Server/Server/StandardServer.cs index e9aedf6b6e..8b67566052 100644 --- a/src/Opc.Ua.Server/Server/StandardServer.cs +++ b/src/Opc.Ua.Server/Server/StandardServer.cs @@ -751,9 +751,8 @@ public override async ValueTask CreateSessionAsync( { string? profile = context.ChannelContext.EndpointDescription!.TransportProfileUri; if (profile is Profiles.UaTcpTransport or Profiles.UaWssTransport && - !Utils.IsEqual( - parsedClientCertificate!.RawData, - context.ChannelContext.ClientChannelCertificate)) + !context.ChannelContext.ClientChannelCertificate.Span.SequenceEqual( + parsedClientCertificate!.RawData)) { throw ServiceResultException.Create( StatusCodes.BadSecurityChecksFailed, diff --git a/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs index 3946f27942..da4e1f6cf4 100644 --- a/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs +++ b/tests/Opc.Ua.Core.Tests/Stack/Server/FairRequestQueueTests.cs @@ -70,6 +70,36 @@ public void WeightedRoundsPreserveOwnerFifo() Assert.That(provider.Used(ResourceIsolationStage.RequestQueueBytes), Is.Zero); } + /// + /// A stopped queue rejects admission before classifying or charging the request. + /// + [Test] + public void StoppedQueueRejectsAdmissionWithoutChargingTheProvider() + { + var provider = new TestProvider(); + NodeId token = provider.AddOwner("A"); + FairRequestQueue queue = CreateQueue(provider); + queue.Dispose(); + + Assert.That(queue.TryEnqueue(new TestRequest(token), default, out StatusCode status), Is.False); + Assert.That(status, Is.EqualTo(StatusCodes.BadServerHalted)); + Assert.That(provider.TotalUsed, Is.Zero); + } + + /// + /// A reader that arrives after the queue stopped is cancelled instead of waiting. + /// + [Test] + public void StoppedQueueCancelsANewReader() + { + var provider = new TestProvider(); + FairRequestQueue queue = CreateQueue(provider); + queue.Dispose(); + + Assert.ThrowsAsync( + async () => await queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false)); + } + /// /// FIFO ordering ignores owner weights and preserves interleaved global admission order. /// diff --git a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs index 66584127de..c50e5819df 100644 --- a/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs +++ b/tests/Opc.Ua.Core.Tests/Types/Utils/ObjectPoolTests.cs @@ -45,6 +45,15 @@ namespace Opc.Ua.Core.Tests.Types.UtilsTests [Parallelizable] public class ObjectPoolTests { + /// + /// A pool must be able to retain at least one object. + /// + [Test] + public void NonPositiveCapacityIsRejected() + { + Assert.Throws(() => _ = new ObjectPool(() => new object(), 0)); + } + /// /// A returned object is handed out again before a new one is created. /// From 30b769d13080ce4d79e67ec0ef2d63ae0b592c62 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 08/19] Server: sample at a fixed rate instead of a fixed delay after each sample The sampling loop waited one interval after every sample, so the timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on Windows) and the sampling time were added to every cycle: a 10 ms group took 63 % (Windows) to 90 % (Linux) of its samples, and the o6-automation opcua-benchmarks subscription test (100 Hz, every item >= 99 % of samples) failed with a single monitored item. Samples now have absolute deadlines one interval apart; a late wake-up shortens the next wait, and a stall longer than four intervals (at least 50 ms) restarts the schedule instead of bursting. The benchmark workload now passes up to 4096 items (about 410k samples/s). NodeState.ReadAttributes fills the result array in place instead of copying a list, removing an allocation per sampled permission check. Co-Authored-By: Claude Opus 5.5 --- .../MonitoredItem/SamplingGroup.cs | 46 +++++++++---- src/Opc.Ua.Types/State/NodeState.cs | 7 +- .../SamplingGroupLoopRegressionTests.cs | 67 +++++++++++++++++++ 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 298259a988..287e5a7ef4 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -570,7 +570,19 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance m_logger.ServerNameThreadStarted(Thread.CurrentThread.Name); int sleepCycle = Convert.ToInt32(samplingInterval, CultureInfo.InvariantCulture); - int timeToWait = sleepCycle; + + // Fixed-rate schedule: each sample has an absolute deadline one interval after + // the previous one. Waiting a fixed interval after each sample instead adds the + // timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on + // Windows) and the sampling time to every cycle, so a 10 ms group sampled about + // 80 to 90 times per second. A late wake-up shortens the next wait, which keeps + // the average rate at the requested interval. + long frequency = m_timeProvider.TimestampFrequency; + long period = Math.Max(1, (long)(samplingInterval * frequency / 1000.0)); + // Missed deadlines are made up by sampling again right away, but not after a + // stall longer than this: the schedule restarts instead of bursting. + long maxLag = Math.Max(4 * period, frequency / 20); + long deadline = m_timeProvider.GetTimestamp() + period; while (m_server.IsRunning && !cancellationToken.IsCancellationRequested) { @@ -579,7 +591,19 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance // wait till next sample without holding a thread. try { - await Task.Delay(timeToWait, cancellationToken).ConfigureAwait(false); + long remaining = deadline - startTimestamp; + if (remaining > 0) + { + await Task.Delay( + TimeSpan.FromTicks(remaining * TimeSpan.TicksPerSecond / frequency), + cancellationToken).ConfigureAwait(false); + } + else + { + // catching up: let other work run between back-to-back samples. + await Task.Yield(); + cancellationToken.ThrowIfCancellationRequested(); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -630,18 +654,14 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance break; } - int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; - timeToWait = sleepCycle; - - if (delay > sleepCycle) + deadline += period; + long now = m_timeProvider.GetTimestamp(); + if (now - deadline > maxLag) { - timeToWait = (2 * sleepCycle) - delay; - - if (timeToWait < 0) - { - m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); - timeToWait = sleepCycle; - } + // the samples take longer than the interval, or the process stalled + int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; + m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); + deadline = now + period; } } diff --git a/src/Opc.Ua.Types/State/NodeState.cs b/src/Opc.Ua.Types/State/NodeState.cs index 0fb8db518f..f8375f415c 100644 --- a/src/Opc.Ua.Types/State/NodeState.cs +++ b/src/Opc.Ua.Types/State/NodeState.cs @@ -4295,7 +4295,8 @@ public virtual ArrayOf ReadAttributes( return []; } - var values = new List(attributeIds.Length); + // filled in place and wrapped without a copy; a list would be copied again. + var values = new Variant[attributeIds.Length]; var scratch = new DataValue(); for (int ii = 0; ii < attributeIds.Length; ii++) @@ -4311,10 +4312,10 @@ public virtual ArrayOf ReadAttributes( default, ref scratch); - values.Add(ServiceResult.IsBad(result) ? default : scratch.WrappedValue); + values[ii] = ServiceResult.IsBad(result) ? default : scratch.WrappedValue; } - return values.ToArrayOf(); + return values; } /// diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index ad603fe015..e6172e1415 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -430,6 +430,73 @@ public async Task CustomItemIsSampledImmediatelyOnlyWithoutQueuedInitialValueAsy } } + /// + /// The loop samples at the requested rate on average. A loop that waits one interval + /// after each sample adds the timer overshoot and the sampling time to every cycle and + /// took 63 % (Windows, 15.6 ms timer) to 90 % (Linux) of the samples of a 10 ms group. + /// + [Test] + [NonParallelizable] + public async Task SamplingKeepsTheRequestedRateAsync() + { + Mock server = DeterministicServerMock.Create( + out MonitoredItemQueueFactory queueFactory); + using (queueFactory) + { + int reads = 0; + var nodeManager = new Mock(); + nodeManager + .Setup(m => m.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Returns, IList, + IList, CancellationToken>( + (_, _, _, values, _, _) => + { + Interlocked.Increment(ref reads); + for (int ii = 0; ii < values.Count; ii++) + { + values[ii] = new DataValue(new Variant(1)); + } + return default; + }); + nodeManager + .Setup(m => m.ValidateRolePermissionsAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(new ValueTask(ServiceResult.Good)); + + using OperationContext context = CreateContext(); + using var group = new SamplingGroup( + server.Object, nodeManager.Object, [new SamplingRateGroup(10, 1, 0)], context, 10); + Mock item = CreateItem(1, 10); + Assert.That(group.StartMonitoring(context, item.Object), Is.True); + group.ApplyChanges(); + + // let the loop start, then count the samples of a fixed window + await Task.Delay(300).ConfigureAwait(false); + int start = Volatile.Read(ref reads); + var window = System.Diagnostics.Stopwatch.StartNew(); + await Task.Delay(2000).ConfigureAwait(false); + int samples = Volatile.Read(ref reads) - start; + double expected = window.Elapsed.TotalMilliseconds / 10; + + group.StopMonitoring(item.Object); + group.ApplyChanges(); + + Assert.That(samples / expected, Is.GreaterThanOrEqualTo(0.95), + $"{samples} samples in {window.Elapsed.TotalMilliseconds:F0} ms at a 10 ms interval"); + Assert.That(samples / expected, Is.LessThanOrEqualTo(1.05), + "the loop must not sample faster than requested"); + } + } + private static int QueuedCount(Mock item) { return item.Invocations.Count(i => i.Method.Name == nameof(IDataChangeMonitoredItem.QueueValue)); From cdac5c5ca4e34461a2a959c053be3a504bdb0175 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 09/19] Server: send responses without blocking pool threads on the channel gate The TCP listeners sent each response through TcpServerChannel.SendResponse, which waits for the channel gate synchronously on the thread-pool worker that finished the request. With many requests in flight per channel, workers parked on the gate while the receive loop that was handed the gate waited for a free pool thread, and the channel stalled until the pool grew: 10 sessions with 100 outstanding Reads each dropped to 0-1.7k req/s (0 with SignAndEncrypt). SendResponseAsync waits for the gate asynchronously; the TCP, Kestrel TCP and WebSocket listeners use it, and the latter two use the cached certificate copies. Same load: 121-131k req/s (89-91k encrypted). Co-Authored-By: Claude Opus 5.5 --- .../Https/HttpsTransportListener.cs | 6 +- .../Tcp/KestrelTcpTransportListener.cs | 8 +- src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs | 37 ++++++ .../Stack/Tcp/TcpTransportListener.cs | 9 +- .../PipelinedRequestTests.cs | 105 ++++++++++++++++++ 5 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs diff --git a/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs b/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs index 0f84429cdc..f08876f2b5 100644 --- a/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs +++ b/src/Opc.Ua.Bindings.Https/Https/HttpsTransportListener.cs @@ -2199,8 +2199,8 @@ private async void OnRequestReceivedAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, peerAddress: (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); @@ -2208,7 +2208,7 @@ private async void OnRequestReceivedAsync( .ProcessRequestAsync(context, request) .ConfigureAwait(false); - serverChannel.SendResponse(requestId, response); + await serverChannel.SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (Exception ex) { diff --git a/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs b/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs index 1b1ffcbac4..64d6a9390c 100644 --- a/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs +++ b/src/Opc.Ua.Bindings.Https/Tcp/KestrelTcpTransportListener.cs @@ -923,14 +923,14 @@ private async Task DispatchRequestAsync( channel.GlobalChannelId, channel.EndpointDescription, RequestEncoding.Binary, - channel.ClientCertificate?.RawData, - channel.ServerCertificate?.RawData, + channel.ClientCertificateRawData, + channel.ServerCertificateRawData, channel.ChannelThumbprint, (channel.Transport?.RemoteEndpoint as IPEndPoint)?.Address); IServiceResponse response = await m_callback .ProcessRequestAsync(context, request) .ConfigureAwait(false); - ((TcpServerChannel)channel).SendResponse(requestId, response); + await ((TcpServerChannel)channel).SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (Exception ex) { @@ -938,7 +938,7 @@ private async Task DispatchRequestAsync( try { ServiceFault fault = EndpointBase.CreateFault(Logger, request, ex); - ((TcpServerChannel)channel).SendResponse(requestId, fault); + await ((TcpServerChannel)channel).SendResponseAsync(requestId, fault).ConfigureAwait(false); } catch (Exception faultEx) { diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs index 73c634eab1..f192694225 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs @@ -1776,6 +1776,43 @@ public bool SendResponse(uint requestId, IServiceResponse response) } using (Gate.Enter()) + { + return SendResponseCore(requestId, response); + } + } + + /// + /// Sends the response for the specified request, waiting for the channel gate + /// without blocking the calling thread. + /// + /// + /// Used by the request dispatch path, which runs on thread-pool workers. With many + /// requests in flight on one channel, the synchronous + /// parked one pool thread per waiting response on the gate; the receive loop that + /// was handed the gate then waited for a free pool thread to continue, and the + /// channel stalled until the pool injected more threads. + /// + /// true if the response was retained for delivery after reconnect. + /// is null. + /// + public async ValueTask SendResponseAsync(uint requestId, IServiceResponse response) + { + if (response == null) + { + throw new ArgumentNullException(nameof(response)); + } + + using (await Gate.EnterAsync().ConfigureAwait(false)) + { + return SendResponseCore(requestId, response); + } + } + + /// + /// Encodes and queues a response while the caller holds the channel gate. + /// + private bool SendResponseCore(uint requestId, IServiceResponse response) + { { // must queue the response if the channel is in the faulted state. if (State == TcpChannelState.Faulted) diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs index b12a02cd54..8bf1306314 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs @@ -1732,7 +1732,8 @@ private async void OnRequestReceivedAsync( try { - responseRetained = ((TcpServerChannel)channel).SendResponse(requestId, response); + responseRetained = await ((TcpServerChannel)channel) + .SendResponseAsync(requestId, response).ConfigureAwait(false); } catch (ServiceResultException sre) when (sre.StatusCode == StatusCodes.BadSecureChannelClosed) { @@ -1749,7 +1750,8 @@ out uint channelId // if the channel is not the same as the one we started with, send the response over the new channel if (serverChannel != channel) { - responseRetained = serverChannel.SendResponse(requestId, response); + responseRetained = await serverChannel + .SendResponseAsync(requestId, response).ConfigureAwait(false); return; } } @@ -1784,7 +1786,8 @@ out uint channelId ServiceFault fault = EndpointBase.CreateFault(m_logger, request, e); (response as IPooledEncodeable)?.Reuse(); response = fault; - responseRetained = ((TcpServerChannel)channel).SendResponse(requestId, fault); + responseRetained = await ((TcpServerChannel)channel) + .SendResponseAsync(requestId, fault).ConfigureAwait(false); } catch (ServiceResultException faultSre) when (faultSre.StatusCode == StatusCodes.BadSecureChannelClosed) diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs new file mode 100644 index 0000000000..145a362137 --- /dev/null +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -0,0 +1,105 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Client; +using Opc.Ua.Client.TestFramework; + +namespace Opc.Ua.Sessions.Tests +{ + /// + /// Many requests in flight on each session complete without stalling the channels. + /// + [TestFixture] + [Category("SessionServices")] + [NonParallelizable] + public class PipelinedRequestTests : TestFixture + { + /// + protected override void ConfigureServer(ApplicationConfiguration configuration) + { + base.ConfigureServer(configuration); + // admit the whole pipeline: 10 sessions x 90 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 2000; + } + + /// + /// The server used to send each response through a synchronous wait on the channel + /// gate from a thread-pool worker. With many responses ready at once, the workers + /// parked on the gate while the receive loop that was handed the gate waited for a + /// free pool thread, and the channel stalled until the pool grew. 90 Reads stay below + /// the channel's limit of 100 queued writes. + /// + [Test] + public async Task ManyConcurrentReadsOnEachSessionCompleteAsync() + { + ArrayOf nodesToRead = new[] + { + new ReadValueId { NodeId = VariableIds.Server_ServerStatus_CurrentTime, AttributeId = Attributes.Value } + }; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var sessions = new List { Session }; + try + { + for (int ii = 1; ii < 10; ii++) + { + sessions.Add(await ClientFixture.ConnectAsync(ServerUrl, SecurityPolicies.None).ConfigureAwait(false)); + } + var elapsed = Stopwatch.StartNew(); + + for (int round = 0; round < 20; round++) + { + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 90).Select( + _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); + ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); + Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); + } + + // 18000 small Reads take under a second when the channels keep flowing; the + // stalled channel took about 10 s while the pool injected threads. + Assert.That(elapsed.Elapsed, Is.LessThan(TimeSpan.FromSeconds(8))); + } + finally + { + foreach (ISession session in sessions.Skip(1)) + { + await session.CloseAsync(5000, true).ConfigureAwait(false); + session.Dispose(); + } + } + } + } +} From 0db74daf425bb55040ef4a94ce6dfdde32c2e1c2 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 10/19] perf: add the counter workload of the o6-automation subscription benchmark `--scenario counters` runs a server whose UInt64 counters advance in their read callback (sampling groups, configurable sampling rate) and checks per item, on the client and on the server, that at least 99 % of the expected samples arrived. `--queue` sizes MaxQueuedRequestCount for deep pipelines. Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/CounterServer.cs | 162 +++++++++++++++++++ perf/ServerLoadHarness/CounterWorkload.cs | 188 ++++++++++++++++++++++ perf/ServerLoadHarness/Program.cs | 33 +++- 3 files changed, 376 insertions(+), 7 deletions(-) create mode 100644 perf/ServerLoadHarness/CounterServer.cs create mode 100644 perf/ServerLoadHarness/CounterWorkload.cs diff --git a/perf/ServerLoadHarness/CounterServer.cs b/perf/ServerLoadHarness/CounterServer.cs new file mode 100644 index 0000000000..4c72ac546e --- /dev/null +++ b/perf/ServerLoadHarness/CounterServer.cs @@ -0,0 +1,162 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Threading; +using Opc.Ua.Server; + +namespace Opc.Ua.Perf.ServerLoadHarness +{ + /// + /// Counter variables advanced by their read callback, so only sampling drives them. + /// Mirrors the subscription capacity workload of the o6-automation opcua-benchmarks + /// (UInt64 counter per monitored item, sampling groups, read-callback increments). + /// + internal sealed class CounterNodeManager : CustomNodeManager2 + { + public const string Namespace = "urn:opcfoundation.org:perf:counters"; + + public CounterNodeManager(IServerInternal server, ApplicationConfiguration configuration, int count) + : base(server, configuration, useSamplingGroups: true, Namespace) + { + m_values = new ulong[count]; + m_marked = new ulong[count]; + } + + public override void CreateAddressSpace(IDictionary> externalReferences) + { + if (!externalReferences.TryGetValue(ObjectIds.ObjectsFolder, out IList? references)) + { + externalReferences[ObjectIds.ObjectsFolder] = references = new List(); + } + for (int i = 0; i < m_values.Length; i++) + { + int index = i; + var node = new BaseDataVariableState(null) + { + NodeId = new NodeId((uint)i + 1, NamespaceIndex), + BrowseName = new QualifiedName("Counter" + i.ToString(CultureInfo.InvariantCulture), NamespaceIndex), + DisplayName = new LocalizedText("Counter" + i.ToString(CultureInfo.InvariantCulture)), + TypeDefinitionId = VariableTypeIds.BaseDataVariableType, + ReferenceTypeId = ReferenceTypeIds.Organizes, + DataType = DataTypeIds.UInt64, + ValueRank = ValueRanks.Scalar, + MinimumSamplingInterval = 1, + AccessLevel = AccessLevels.CurrentRead, + UserAccessLevel = AccessLevels.CurrentRead, + Value = new Variant((ulong)0), + StatusCode = StatusCodes.Good, + Timestamp = DateTimeUtc.Now + }; + node.OnReadValue = (ISystemContext context, NodeState state, NumericRange range, + QualifiedName encoding, ref Variant value, ref StatusCode status, ref DateTimeUtc timestamp) => + { + value = new Variant(Interlocked.Increment(ref m_values[index])); + status = StatusCodes.Good; + timestamp = DateTimeUtc.Now; + return ServiceResult.Good; + }; + node.AddReference(ReferenceTypeIds.Organizes, true, ObjectIds.ObjectsFolder); + references.Add(new NodeStateReference(ReferenceTypeIds.Organizes, false, node.NodeId)); + AddPredefinedNode(SystemContext, node); + } + } + + /// + /// Starts a measurement window. + /// + public void Mark() + { + for (int i = 0; i < m_values.Length; i++) + { + m_marked[i] = Interlocked.Read(ref m_values[i]); + } + m_markTimestamp = Stopwatch.GetTimestamp(); + } + + /// + /// Reports the sampling progress of the window: completed read callbacks per item + /// against the number the sampling interval calls for. + /// + public string Report(double samplingIntervalMs) + { + double elapsedMs = Stopwatch.GetElapsedTime(m_markTimestamp).TotalMilliseconds; + double expected = elapsedMs / samplingIntervalMs; + double min = double.MaxValue; + double sum = 0; + int belowTarget = 0; + for (int i = 0; i < m_values.Length; i++) + { + double progress = (Interlocked.Read(ref m_values[i]) - m_marked[i]) / expected; + min = Math.Min(min, progress); + sum += progress; + if (progress < 0.99) + { + belowTarget++; + } + } + return FormattableString.Invariant( + $"counters={m_values.Length} samplesExpected={expected:F0} progressMin={min:P2} ") + + FormattableString.Invariant( + $"progressMean={sum / m_values.Length:P2} itemsBelow99={belowTarget}"); + } + + private readonly ulong[] m_values; + private readonly ulong[] m_marked; + private long m_markTimestamp = Stopwatch.GetTimestamp(); + } + + /// + /// A server exposing only the counter node manager in addition to the standard nodes. + /// + internal sealed class CounterServer : StandardServer + { + public CounterServer(ITelemetryContext telemetry, int count) + : base(telemetry) + { + m_count = count; + } + + public CounterNodeManager? Counters { get; private set; } + + protected override IMasterNodeManager CreateMasterNodeManager( + IServerInternal server, + ApplicationConfiguration configuration) + { + Counters = new CounterNodeManager(server, configuration, m_count); + return new MasterNodeManager(server, configuration, null, Counters); + } + + private readonly int m_count; + } +} diff --git a/perf/ServerLoadHarness/CounterWorkload.cs b/perf/ServerLoadHarness/CounterWorkload.cs new file mode 100644 index 0000000000..a4f6bbd7cb --- /dev/null +++ b/perf/ServerLoadHarness/CounterWorkload.cs @@ -0,0 +1,188 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Client; + +namespace Opc.Ua.Perf.ServerLoadHarness +{ + /// + /// Client observer of the counter workload: one subscription with a monitored item per + /// counter (queue size one, discard oldest). A counter advances once per server sample, + /// so its progress over the window, against window / sampling interval, is the share of + /// samples the server took. Every item must reach 99% to pass, as in the o6-automation + /// opcua-benchmarks subscription suite. + /// + internal sealed class CounterWorkload : Workload + { + public CounterWorkload(ISession session, int count, int samplingInterval, int publishingInterval) + { + m_session = session; + m_samplingInterval = samplingInterval; + m_publishingInterval = publishingInterval; + m_latestValue = new ulong[count]; + m_latestTicks = new long[count]; + m_startValue = new ulong[count]; + m_startTicks = new long[count]; + } + + public override double Operations => Interlocked.Read(ref m_notifications); + + public override string OperationName => "notification"; + + public override async Task SetupAsync() + { + ushort namespaceIndex = (ushort)m_session.NamespaceUris.GetIndex(CounterNodeManager.Namespace); +#pragma warning disable CA2000 // ownership transfers to the session in AddSubscription; disposed below on failure before that + var subscription = new Subscription(m_session.DefaultSubscription) + { + PublishingInterval = m_publishingInterval, + KeepAliveCount = 10, + LifetimeCount = 10000, + MaxNotificationsPerPublish = 0, + TimestampsToReturn = TimestampsToReturn.Neither, + FastDataChangeCallback = OnDataChange + }; +#pragma warning restore CA2000 + try + { + for (int i = 0; i < m_latestValue.Length; i++) + { + var item = new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = new NodeId((uint)i + 1, namespaceIndex), + AttributeId = Attributes.Value, + MonitoringMode = MonitoringMode.Reporting, + SamplingInterval = m_samplingInterval, + QueueSize = 1, + DiscardOldest = true + }; + m_indexByHandle[item.ClientHandle] = i; + subscription.AddItem(item); + } + // the session owns and disposes the subscription from here on + m_session.AddSubscription(subscription); + } + catch + { + subscription.Dispose(); + throw; + } + var setup = Stopwatch.StartNew(); + await subscription.CreateAsync().ConfigureAwait(false); + Console.WriteLine(FormattableString.Invariant( + $"counters: {m_latestValue.Length} items, sampling {m_samplingInterval} ms, ") + + FormattableString.Invariant( + $"publishing {m_publishingInterval} ms, revised sampling {System.Linq.Enumerable.FirstOrDefault(subscription.MonitoredItems)?.Status.SamplingInterval}; setup {setup.ElapsedMilliseconds} ms")); + } + + public override async Task RunAsync(CancellationToken ct) + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + + public override void ResetStats() + { + lock (m_lock) + { + Array.Copy(m_latestValue, m_startValue, m_latestValue.Length); + Array.Copy(m_latestTicks, m_startTicks, m_latestTicks.Length); + } + Interlocked.Exchange(ref m_notifications, 0); + } + + public override string Report(double seconds) + { + double min = double.MaxValue; + double sum = 0; + int below = 0; + int missing = 0; + lock (m_lock) + { + for (int i = 0; i < m_latestValue.Length; i++) + { + if (m_startTicks[i] == 0 || m_latestTicks[i] <= m_startTicks[i]) + { + missing++; + min = 0; + continue; + } + double elapsedMs = (m_latestTicks[i] - m_startTicks[i]) * 1000.0 / Stopwatch.Frequency; + double progress = (m_latestValue[i] - m_startValue[i]) / (elapsedMs / m_samplingInterval); + min = Math.Min(min, progress); + sum += progress; + if (progress < 0.99) + { + below++; + } + } + } + int measured = m_latestValue.Length - missing; + return FormattableString.Invariant( + $"notif/s={Interlocked.Read(ref m_notifications) / seconds:F0} progressMin={min:P2} ") + + FormattableString.Invariant( + $"progressMean={(measured == 0 ? 0 : sum / measured):P2} itemsBelow99={below} itemsWithoutData={missing} ") + + (below == 0 && missing == 0 ? "PASS" : "FAIL"); + } + + private void OnDataChange(Subscription subscription, DataChangeNotification notification, ArrayOf stringTable) + { + long now = Stopwatch.GetTimestamp(); + lock (m_lock) + { + foreach (MonitoredItemNotification item in notification.MonitoredItems) + { + if (m_indexByHandle.TryGetValue(item.ClientHandle, out int index) && + item.Value.WrappedValue.TryGetValue(out ulong value)) + { + m_latestValue[index] = value; + m_latestTicks[index] = now; + } + } + } + Interlocked.Add(ref m_notifications, notification.MonitoredItems.Count); + } + + private readonly ISession m_session; + private readonly int m_samplingInterval; + private readonly int m_publishingInterval; + private readonly Dictionary m_indexByHandle = []; + private readonly ulong[] m_latestValue; + private readonly long[] m_latestTicks; + private readonly ulong[] m_startValue; + private readonly long[] m_startTicks; + private readonly Lock m_lock = new(); + private long m_notifications; + } +} diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index 320d3807ba..b7731f46f7 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -177,6 +177,21 @@ private static async Task RunServerAsync(ApplicationInstance application, I config.ServerConfiguration.MaxFailedAuthenticationAttempts = 0; config.ServerConfiguration.MaxRequestThreadCount = ArgInt("threads", 200); config.ServerConfiguration.MinRequestThreadCount = 50; + // sized for the largest pipeline (sessions x inflight), as the o6 benchmarks do + config.ServerConfiguration.MaxQueuedRequestCount = ArgInt("queue", 2000); + int counters = ArgInt("counters", 0); + int sampling = ArgInt("sampling", 10); + if (counters > 0) + { + // the subscription capacity workload of o6-automation/opcua-benchmarks + config.ServerConfiguration.MinPublishingInterval = 1; + config.ServerConfiguration.PublishingResolution = 1; + config.ServerConfiguration.MaxPublishingInterval = 86400_000; + config.ServerConfiguration.MaxNotificationQueueSize = 1; + config.ServerConfiguration.MaxNotificationsPerPublish = int.MaxValue; + config.ServerConfiguration.MaxPublishRequestCount = 4; + config.ServerConfiguration.AvailableSamplingRates = [new SamplingRateGroup(sampling, 1, 0)]; + } if (!await application .CheckApplicationInstanceCertificatesAsync(true, CertificateFactory.DefaultLifeTime) @@ -186,10 +201,10 @@ private static async Task RunServerAsync(ApplicationInstance application, I return 1; } - using var server = new ReferenceServer(telemetry) - { - TransportBindings = TestTransportBindings.WithAllSchemes() - }; + using Opc.Ua.Server.StandardServer server = counters > 0 + ? new CounterServer(telemetry, counters) + : new ReferenceServer(telemetry); + server.TransportBindings = TestTransportBindings.WithAllSchemes(); await application.StartAsync(server).ConfigureAwait(false); Console.WriteLine(FormattableString.Invariant($"READY {endpointUrl} {Environment.ProcessId}")); @@ -203,10 +218,12 @@ private static async Task RunServerAsync(ApplicationInstance application, I GC.Collect(); GC.WaitForPendingFinalizers(); mark = ServerStats.Capture(); + (server as CounterServer)?.Counters?.Mark(); Console.WriteLine("MARKED"); break; case "report": - Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + string counterReport = (server as CounterServer)?.Counters?.Report(sampling) ?? string.Empty; + Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark) + " " + counterReport); break; case "quit": await server.StopAsync().ConfigureAwait(false); @@ -228,7 +245,7 @@ private static async Task RunDriverAsync(string[] args) var startInfo = new ProcessStartInfo( Environment.ProcessPath!, FormattableString.Invariant( - $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")}")) + $"server --port {port} --diag {Arg("diag", "0")} --audit {Arg("audit", "0")} --threads {Arg("threads", "200")} --counters {(scenario == "counters" ? Arg("nodes", "1") : "0")} --sampling {Arg("sampling", "10")} --queue {Arg("queue", "2000")}")) { RedirectStandardInput = true, RedirectStandardOutput = true, @@ -290,7 +307,9 @@ await Parallel.ForAsync( Console.WriteLine(FormattableString.Invariant( $"connected {sessionCount} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); - Workload workload = scenario == "sub" + Workload workload = scenario == "counters" + ? new CounterWorkload(sessions[0], ArgInt("nodes", 1), ArgInt("sampling", 10), ArgInt("pub", 1000)) + : scenario == "sub" ? new SubscribeWorkload( sessions, ArgInt("nodes", 100), From 6faf39ef19b5a11d6d57572b54ef21a509e81d9e Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:52:28 +0200 Subject: [PATCH 11/19] Reference server: fix wording of the Server GC comment Co-Authored-By: Claude Opus 5.5 --- .../ConsoleReferenceServer/ConsoleReferenceServer.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj index 1f36086530..6bf132e7ce 100644 --- a/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj +++ b/samples/Reference/ConsoleReferenceServer/ConsoleReferenceServer.csproj @@ -13,7 +13,7 @@ true true From 086554371674f9576b4357d2899e4a788705e34b Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:55:57 +0200 Subject: [PATCH 12/19] Server: apply backpressure instead of faulting a channel with a full write queue A server channel was faulted with BadTcpNotEnoughResources ("The channel write queue is full") when a request arrived while more than 100 responses were queued for writing, so a client pipelining more requests than that per channel (e.g. the 1000 outstanding Reads per client of the o6-automation benchmark) lost its channel. Server channels now stop reading while the queue is full and resume once it has drained to 50; TCP flow control slows the client instead. Requests already read when the queue filled stay bounded by the request queue limits. Client channels keep reading, because responses are what drain their writes. 10 clients x 1000 outstanding Reads: channels faulted before, now 0 errors at about 45-54k req/s. Co-Authored-By: Claude Opus 5.5 --- src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs | 23 ++--- .../Stack/Tcp/UaSCBinaryChannel.cs | 85 ++++++++++++++++++- .../PipelinedRequestTests.cs | 46 +++++++++- 3 files changed, 134 insertions(+), 20 deletions(-) diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs index f192694225..f2d0dc81e0 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpServerChannel.cs @@ -45,6 +45,9 @@ namespace Opc.Ua.Bindings /// public class TcpServerChannel : TcpListenerChannel { + /// + protected override bool AppliesReceiveBackpressure => true; + /// /// Attaches the object to an existing socket. /// @@ -1586,15 +1589,9 @@ private bool ProcessRequestMessage( return false; } - if (ChannelFull) - { - m_logger.TcpServerLog13(Id); - m_logger.TcpServerLog14(Id); - ForceChannelFaultCore( - StatusCodes.BadTcpNotEnoughResources, - "The channel write queue is full."); - return false; - } + // A full write queue no longer faults the channel: the receive loop stops + // reading until responses drain (see AppliesReceiveBackpressure). Requests + // already read when the queue filled are bounded by the request queue limits. BufferCollection? chunksToProcess = null; @@ -2105,14 +2102,6 @@ public static partial void TcpServerLog12( uint currentToken, uint previousToken); - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 13, Level = LogLevel.Information, - Message = "Channel {Id}: full -- delay processing.")] - public static partial void TcpServerLog13(this ILogger logger, uint id); - - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 14, Level = LogLevel.Warning, - Message = "Channel {Id}: break socket connection.")] - public static partial void TcpServerLog14(this ILogger logger, uint id); - [LoggerMessage(EventId = CoreEventIds.TcpServerChannel + 15, Level = LogLevel.Warning, Message = "ChannelId {Id}: ProcessRequestMessage RequestId {RequestId} was aborted.")] public static partial void TcpServerLog15( diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs index adf4af4461..a73f8375c5 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs @@ -965,7 +965,84 @@ private static void DisposeIsolationLeases(List? leases) } /// - public virtual bool ChannelFull => m_activeWriteRequests > 100; + public virtual bool ChannelFull => Volatile.Read(ref m_activeWriteRequests) > 100; + + /// + /// Whether the receive loop stops reading while the write queue is full. + /// + /// + /// A server channel only writes responses to requests it read, so not reading + /// lets TCP flow control slow a client that pipelines more requests than the + /// responses it consumes. A client channel must keep reading: its writes are + /// requests, and responses are what drains them. + /// + protected virtual bool AppliesReceiveBackpressure => false; + + /// + /// Completes when the write queue has room again, or right away when it has. + /// + private ValueTask WaitForWriteCapacityAsync(CancellationToken ct) + { + return AppliesReceiveBackpressure && ChannelFull + ? WaitForWriteCapacitySlowAsync(ct) + : default; + } + + private async ValueTask WaitForWriteCapacitySlowAsync(CancellationToken ct) + { + // resume only once the queue has drained to the low-water mark, so the loop + // reads a batch per wake-up instead of one request per completed write. + while (!HasWriteCapacityToResume) + { + TaskCompletionSource waiter; + lock (m_writeCapacityLock) + { + waiter = m_writeCapacityWaiter ??= + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + // re-checked after publishing the waiter: a write that completed in + // between saw no waiter and did not signal. + if (HasWriteCapacityToResume) + { + return; + } + + using (ct.Register(static state => ((TaskCompletionSource)state!).TrySetCanceled(), waiter)) + { + await waiter.Task.ConfigureAwait(false); + } + } + } + + /// + /// Whether a receive loop paused by a full write queue may read again. + /// + private bool HasWriteCapacityToResume => + !ChannelFull && Volatile.Read(ref m_activeWriteRequests) <= kResumeWriteCount; + + /// + /// The queued-write count a paused receive loop resumes at (half the full mark). + /// + private const int kResumeWriteCount = 50; + + /// + /// Releases a receive loop that waits for write capacity once the queue has room. + /// + private void SignalWriteCapacity() + { + if (Volatile.Read(ref m_writeCapacityWaiter) == null || !HasWriteCapacityToResume) + { + return; + } + TaskCompletionSource? waiter; + lock (m_writeCapacityLock) + { + waiter = m_writeCapacityWaiter; + m_writeCapacityWaiter = null; + } + waiter?.TrySetResult(true); + } /// /// Indicates that admission cleanup must preserve the channel until outstanding writes finish. @@ -1253,6 +1330,9 @@ private async Task RunReceiveLoopAsync(IUaSCByteTransport transport, Cancellatio ArraySegment chunk; try { + // backpressure: do not read more requests while their responses + // cannot be written; the peer's sends block in TCP flow control. + await WaitForWriteCapacityAsync(ct).ConfigureAwait(false); chunk = await transport.ReceiveChunkAsync(ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) @@ -1586,6 +1666,7 @@ protected virtual void HandleWriteComplete( buffers?.Release(BufferManager, "WriteOperation"); Interlocked.Decrement(ref m_activeWriteRequests); + SignalWriteCapacity(); } /// @@ -1979,6 +2060,8 @@ public void UpdateLastActiveTime() /// private int m_state; private int m_activeWriteRequests; + private readonly Lock m_writeCapacityLock = new(); + private TaskCompletionSource? m_writeCapacityWaiter; private readonly Lock m_writeChainLock = new(); private Task m_writeChain = Task.CompletedTask; private long m_lastActiveTimestamp; diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs index 145a362137..33d2060309 100644 --- a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -52,8 +52,50 @@ public class PipelinedRequestTests : TestFixture protected override void ConfigureServer(ApplicationConfiguration configuration) { base.ConfigureServer(configuration); - // admit the whole pipeline: 10 sessions x 90 requests - configuration.ServerConfiguration!.MaxQueuedRequestCount = 2000; + // admit the whole pipeline: up to 10 sessions x 300 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 4000; + } + + /// + /// More requests in flight on a session than the channel's write queue holds (100) + /// slow the client down instead of closing its channel: the server stops reading + /// while the responses drain. The channel used to be faulted with + /// BadTcpNotEnoughResources ("The channel write queue is full"). + /// + [Test] + public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() + { + ArrayOf nodesToRead = new[] + { + new ReadValueId { NodeId = VariableIds.Server_ServerStatus_CurrentTime, AttributeId = Attributes.Value } + }; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var sessions = new List { Session }; + try + { + for (int ii = 1; ii < 4; ii++) + { + sessions.Add(await ClientFixture.ConnectAsync(ServerUrl, SecurityPolicies.None).ConfigureAwait(false)); + } + + for (int round = 0; round < 5; round++) + { + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 300).Select( + _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); + ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); + Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); + } + + Assert.That(sessions.All(session => session.Connected), Is.True); + } + finally + { + foreach (ISession session in sessions.Skip(1)) + { + await session.CloseAsync(5000, true).ConfigureAwait(false); + session.Dispose(); + } + } } /// From 191cd2ba787258c4c164ff9ea213274a45194d28 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 09:55:57 +0200 Subject: [PATCH 13/19] perf: report client CPU for the measurement window Co-Authored-By: Claude Opus 5.5 --- perf/ServerLoadHarness/Program.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index b7731f46f7..eda23c12d9 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -333,10 +333,14 @@ await Parallel.ForAsync( Take(serverLines, "MARKED"); workload.ResetStats(); var window = Stopwatch.StartNew(); + TimeSpan clientCpuStart = Process.GetCurrentProcess().TotalProcessorTime; await Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ConfigureAwait(false); await server.StandardInput.WriteLineAsync("report").ConfigureAwait(false); string report = Take(serverLines, "REPORT")[7..]; - string clientReport = workload.Report(window.Elapsed.TotalSeconds); + double clientCores = (Process.GetCurrentProcess().TotalProcessorTime - clientCpuStart).TotalSeconds / + window.Elapsed.TotalSeconds; + string clientReport = workload.Report(window.Elapsed.TotalSeconds) + + FormattableString.Invariant($" clientCpuCores={clientCores:F2}"); await stop.CancelAsync().ConfigureAwait(false); try { From 10292b1eb307d3492f2ac12dd87fd5747320773a Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 10:40:15 +0200 Subject: [PATCH 14/19] Tests: run the deep-pipeline test with 1000 Reads per session In process the client drains responses fast enough that the server's write queue rarely fills, so the test checks that deep pipelines complete with the channels connected; the out-of-process harness exercises the full queue. Co-Authored-By: Claude Opus 5.5 --- .../Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs index 33d2060309..4c5da45c56 100644 --- a/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs +++ b/tests/Opc.Ua.Sessions.Tests/PipelinedRequestTests.cs @@ -52,15 +52,17 @@ public class PipelinedRequestTests : TestFixture protected override void ConfigureServer(ApplicationConfiguration configuration) { base.ConfigureServer(configuration); - // admit the whole pipeline: up to 10 sessions x 300 requests - configuration.ServerConfiguration!.MaxQueuedRequestCount = 4000; + // admit the whole pipeline: up to 4 sessions x 1000 requests + configuration.ServerConfiguration!.MaxQueuedRequestCount = 10000; } /// /// More requests in flight on a session than the channel's write queue holds (100) - /// slow the client down instead of closing its channel: the server stops reading - /// while the responses drain. The channel used to be faulted with - /// BadTcpNotEnoughResources ("The channel write queue is full"). + /// complete and leave the channel connected. When responses back up, the server + /// stops reading until they drain instead of faulting the channel with + /// BadTcpNotEnoughResources ("The channel write queue is full"). In process the + /// client reads fast enough that the queue rarely fills; the out-of-process load + /// harness (perf/ServerLoadHarness, 10 clients x 1000 Reads) exercises the full queue. /// [Test] public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() @@ -80,7 +82,7 @@ public async Task PipelinesDeeperThanTheWriteQueueAreThrottledNotFaultedAsync() for (int round = 0; round < 5; round++) { - IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 300).Select( + IEnumerable> reads = sessions.SelectMany(session => Enumerable.Range(0, 1000).Select( _ => session.ReadAsync(null, 0, TimestampsToReturn.Neither, nodesToRead, deadline.Token).AsTask())); ReadResponse[] responses = await Task.WhenAll(reads).ConfigureAwait(false); Assert.That(responses.All(r => StatusCode.IsGood(r.Results[0].StatusCode)), Is.True); From d8bdd0295b999aded0601da6ee6f5c7cf7b778e5 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 10:55:04 +0200 Subject: [PATCH 15/19] Server: test the sampling schedule deterministically; tolerate 500 ms of lag SamplingKeepsTheRequestedRateAsync measured the sampling rate against the wall clock and failed on a loaded macOS CI runner (78 % of the samples). The deadline arithmetic now lives in SamplingSchedule, and the tests drive it with simulated clocks: Windows- and Linux-like timer overshoot keeps the rate (where waiting a fixed interval drifts), a 200 ms pause is made up, a 3 s stall restarts the schedule once, and sampling slower than the interval does not run away. The schedule now tolerates a lag of four intervals or 500 ms (was 50 ms) before restarting, so a garbage collection or scheduling pause of that length is made up rather than lost; on a loaded machine this kept more samples at 4096 items (81 % vs 62-71 %). Co-Authored-By: Claude Opus 5.5 --- .../MonitoredItem/SamplingGroup.cs | 19 +- .../MonitoredItem/SamplingSchedule.cs | 109 +++++++++++ .../SamplingGroupLoopRegressionTests.cs | 169 ++++++++++++------ 3 files changed, 225 insertions(+), 72 deletions(-) create mode 100644 src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 287e5a7ef4..5d0750ee8c 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -571,18 +571,8 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance int sleepCycle = Convert.ToInt32(samplingInterval, CultureInfo.InvariantCulture); - // Fixed-rate schedule: each sample has an absolute deadline one interval after - // the previous one. Waiting a fixed interval after each sample instead adds the - // timer overshoot (about 1 ms on Linux, up to the 15.6 ms timer resolution on - // Windows) and the sampling time to every cycle, so a 10 ms group sampled about - // 80 to 90 times per second. A late wake-up shortens the next wait, which keeps - // the average rate at the requested interval. long frequency = m_timeProvider.TimestampFrequency; - long period = Math.Max(1, (long)(samplingInterval * frequency / 1000.0)); - // Missed deadlines are made up by sampling again right away, but not after a - // stall longer than this: the schedule restarts instead of bursting. - long maxLag = Math.Max(4 * period, frequency / 20); - long deadline = m_timeProvider.GetTimestamp() + period; + var schedule = SamplingSchedule.Create(samplingInterval, frequency, m_timeProvider.GetTimestamp()); while (m_server.IsRunning && !cancellationToken.IsCancellationRequested) { @@ -591,7 +581,7 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance // wait till next sample without holding a thread. try { - long remaining = deadline - startTimestamp; + long remaining = schedule.GetWait(startTimestamp); if (remaining > 0) { await Task.Delay( @@ -654,14 +644,11 @@ await Task.Delay( break; } - deadline += period; - long now = m_timeProvider.GetTimestamp(); - if (now - deadline > maxLag) + if (schedule.Advance(m_timeProvider.GetTimestamp())) { // the samples take longer than the interval, or the process stalled int delay = (int)m_timeProvider.GetElapsedTime(startTimestamp).TotalMilliseconds; m_logger.WARNINGSamplingGroupCannotSampleFastEnoughTimeToSample(delay, sleepCycle); - deadline = now + period; } } diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs new file mode 100644 index 0000000000..16d0ca87b7 --- /dev/null +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs @@ -0,0 +1,109 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + + +using System; + +namespace Opc.Ua.Server +{ + /// + /// The fixed-rate schedule of a sampling group, in timestamp ticks. + /// + /// + /// Each sample has an absolute deadline one interval after the previous one. Waiting a + /// fixed interval after each sample instead adds the timer overshoot (about 1 ms on + /// Linux, up to the 15.6 ms timer resolution on Windows) and the sampling time to every + /// cycle, so a 10 ms group sampled about 63 to 90 times per second. With deadlines a + /// late wake-up shortens the next wait, which keeps the average rate at the requested + /// interval. Missed deadlines are made up by sampling again right away, but not after a + /// stall longer than : the schedule then restarts from the current + /// time instead of bursting. + /// + internal struct SamplingSchedule + { + /// + /// Creates a schedule whose first sample is one period after . + /// + public SamplingSchedule(long now, long period, long maxLag) + { + Period = Math.Max(1, period); + MaxLag = maxLag; + Deadline = now + Period; + } + + /// + /// The ticks between two samples. + /// + public long Period { get; } + + /// + /// How far the schedule may fall behind before it restarts. + /// + public long MaxLag { get; } + + /// + /// The timestamp the next sample is due at. + /// + public long Deadline { get; private set; } + + /// + /// Creates the schedule of a sampling interval in milliseconds. It tolerates a lag of + /// four intervals or half a second, whichever is longer, so a garbage collection or + /// scheduling pause of that length is made up rather than lost. + /// + public static SamplingSchedule Create(double samplingIntervalMs, long frequency, long now) + { + long period = Math.Max(1, (long)(samplingIntervalMs * frequency / 1000.0)); + return new SamplingSchedule(now, period, Math.Max(4 * period, frequency / 2)); + } + + /// + /// The ticks to wait before the next sample; zero or less when it is due. + /// + public readonly long GetWait(long now) + { + return Deadline - now; + } + + /// + /// Moves to the deadline after the sample just taken. + /// + /// true when the schedule had fallen too far behind and restarted. + public bool Advance(long now) + { + Deadline += Period; + if (now - Deadline > MaxLag) + { + Deadline = now + Period; + return true; + } + return false; + } + } +} diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index e6172e1415..a1968ddf3c 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -431,69 +431,126 @@ public async Task CustomItemIsSampledImmediatelyOnlyWithoutQueuedInitialValueAsy } /// - /// The loop samples at the requested rate on average. A loop that waits one interval - /// after each sample adds the timer overshoot and the sampling time to every cycle and - /// took 63 % (Windows, 15.6 ms timer) to 90 % (Linux) of the samples of a 10 ms group. + /// A timer that wakes up late on every wait does not slow the schedule down: with a + /// Windows-like 5.6 ms overshoot on a 10 ms interval the schedule still takes a sample + /// every 10 ms on average, where waiting one interval after each sample takes 64 %. /// - [Test] - [NonParallelizable] - public async Task SamplingKeepsTheRequestedRateAsync() + [TestCase(5.6)] + [TestCase(1.0)] + [TestCase(0.0)] + public void ScheduleKeepsTheRateWhenTheTimerOvershoots(double overshootMs) { - Mock server = DeterministicServerMock.Create( - out MonitoredItemQueueFactory queueFactory); - using (queueFactory) + int samples = SimulateSchedule(10, overshootMs, sampleCostMs: 0.3, seconds: 10, out int restarts); + int fixedDelaySamples = SimulateFixedDelay(10, overshootMs, sampleCostMs: 0.3, seconds: 10); + + Assert.That(samples, Is.InRange(995, 1001)); + Assert.That(restarts, Is.Zero); + if (overshootMs > 0) { - int reads = 0; - var nodeManager = new Mock(); - nodeManager - .Setup(m => m.ReadAsync( - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>(), - It.IsAny>(), - It.IsAny())) - .Returns, IList, - IList, CancellationToken>( - (_, _, _, values, _, _) => - { - Interlocked.Increment(ref reads); - for (int ii = 0; ii < values.Count; ii++) - { - values[ii] = new DataValue(new Variant(1)); - } - return default; - }); - nodeManager - .Setup(m => m.ValidateRolePermissionsAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(new ValueTask(ServiceResult.Good)); + Assert.That(fixedDelaySamples, Is.LessThan(975), "the simulation must reproduce the drift"); + } + } - using OperationContext context = CreateContext(); - using var group = new SamplingGroup( - server.Object, nodeManager.Object, [new SamplingRateGroup(10, 1, 0)], context, 10); - Mock item = CreateItem(1, 10); - Assert.That(group.StartMonitoring(context, item.Object), Is.True); - group.ApplyChanges(); + /// + /// A pause shorter than the tolerated lag, such as a garbage collection, is made up by + /// sampling again right away, so no sample of the interval is lost. + /// + [Test] + public void SchedulePausesShorterThanTheLagAreMadeUp() + { + int samples = SimulateSchedule(10, 1.0, 0.3, 10, out int restarts, stallAtMs: 5000, stallMs: 200); - // let the loop start, then count the samples of a fixed window - await Task.Delay(300).ConfigureAwait(false); - int start = Volatile.Read(ref reads); - var window = System.Diagnostics.Stopwatch.StartNew(); - await Task.Delay(2000).ConfigureAwait(false); - int samples = Volatile.Read(ref reads) - start; - double expected = window.Elapsed.TotalMilliseconds / 10; + Assert.That(samples, Is.InRange(995, 1001)); + Assert.That(restarts, Is.Zero); + } - group.StopMonitoring(item.Object); - group.ApplyChanges(); + /// + /// A stall longer than the tolerated lag restarts the schedule once instead of bursting + /// through every missed sample. + /// + [Test] + public void ScheduleRestartsAfterAStallLongerThanTheLag() + { + int samples = SimulateSchedule(10, 1.0, 0.3, 10, out int restarts, stallAtMs: 5000, stallMs: 3000); + + Assert.That(restarts, Is.EqualTo(1)); + Assert.That(samples, Is.InRange(690, 710), "the 3 s stall is skipped, not made up"); + } + + /// + /// Samples that take longer than the interval run back to back without the schedule + /// running away: it restarts whenever it falls a full lag behind. + /// + [Test] + public void ScheduleDoesNotRunAwayWhenSamplingIsSlowerThanTheInterval() + { + int samples = SimulateSchedule(10, 0.0, 15, 10, out int restarts); + + Assert.That(samples, Is.InRange(660, 670)); + Assert.That(restarts, Is.GreaterThan(0)); + } + + private const long kTicksPerSecond = 10_000_000; + + private static long Ticks(double milliseconds) + { + return (long)(milliseconds * kTicksPerSecond / 1000); + } + + private static int SimulateSchedule( + double intervalMs, + double overshootMs, + double sampleCostMs, + double seconds, + out int restarts, + double stallAtMs = -1, + double stallMs = 0) + { + long now = 0; + long end = Ticks(seconds * 1000); + bool stalled = false; + var schedule = SamplingSchedule.Create(intervalMs, kTicksPerSecond, now); + int samples = 0; + restarts = 0; + while (true) + { + long wait = schedule.GetWait(now); + if (wait > 0) + { + now += wait + Ticks(overshootMs); + } + if (!stalled && stallAtMs >= 0 && now >= Ticks(stallAtMs)) + { + stalled = true; + now += Ticks(stallMs); + } + if (now >= end) + { + return samples; + } + now += Ticks(sampleCostMs); + samples++; + if (schedule.Advance(now)) + { + restarts++; + } + } + } - Assert.That(samples / expected, Is.GreaterThanOrEqualTo(0.95), - $"{samples} samples in {window.Elapsed.TotalMilliseconds:F0} ms at a 10 ms interval"); - Assert.That(samples / expected, Is.LessThanOrEqualTo(1.05), - "the loop must not sample faster than requested"); + private static int SimulateFixedDelay(double intervalMs, double overshootMs, double sampleCostMs, double seconds) + { + long now = 0; + long end = Ticks(seconds * 1000); + int samples = 0; + while (true) + { + now += Ticks(intervalMs + overshootMs); + if (now >= end) + { + return samples; + } + now += Ticks(sampleCostMs); + samples++; } } From f2df124124024c3cc1974e67bd6b76a86571c19c Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 11:10:57 +0200 Subject: [PATCH 16/19] Fix sampling delay overflow for long intervals; await snapshot test readers The wait before a sample was converted with remaining * TicksPerSecond / frequency, which overflows a long for long intervals at the 1 GHz timestamp frequency of Unix (one hour is 3.6e19); Task.Delay then threw and ended the sampling loop. SamplingSchedule.ToTimeSpan scales through a double, covered by a one-hour test at 1 GHz. ResourceManagerSnapshotTests awaits its readers with a timeout instead of blocking on Task.WaitAll. Co-Authored-By: Claude Opus 5.5 --- .../NodeManager/MonitoredItem/SamplingGroup.cs | 2 +- .../NodeManager/MonitoredItem/SamplingSchedule.cs | 12 ++++++++++++ .../SamplingGroupLoopRegressionTests.cs | 15 +++++++++++++++ .../ResourceManagerSnapshotTests.cs | 7 ++++--- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs index 5d0750ee8c..fee3983f3a 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingGroup.cs @@ -585,7 +585,7 @@ private async ValueTask SampleMonitoredItemsAsync(double samplingInterval, Cance if (remaining > 0) { await Task.Delay( - TimeSpan.FromTicks(remaining * TimeSpan.TicksPerSecond / frequency), + SamplingSchedule.ToTimeSpan(remaining, frequency), cancellationToken).ConfigureAwait(false); } else diff --git a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs index 16d0ca87b7..3bd27f40e5 100644 --- a/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs +++ b/src/Opc.Ua.Server/NodeManager/MonitoredItem/SamplingSchedule.cs @@ -83,6 +83,18 @@ public static SamplingSchedule Create(double samplingIntervalMs, long frequency, return new SamplingSchedule(now, period, Math.Max(4 * period, frequency / 2)); } + /// + /// Converts a span of timestamp ticks to a . + /// + /// + /// Scaled through a double: multiplying first overflows a long for long intervals at + /// a 1 GHz timestamp frequency (Unix), e.g. one hour is 3.6e12 ticks times 1e7. + /// + public static TimeSpan ToTimeSpan(long timestampTicks, long frequency) + { + return TimeSpan.FromTicks((long)(timestampTicks * ((double)TimeSpan.TicksPerSecond / frequency))); + } + /// /// The ticks to wait before the next sample; zero or less when it is due. /// diff --git a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs index a1968ddf3c..c42fe7de08 100644 --- a/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs +++ b/tests/Opc.Ua.Server.Tests/NodeManager/SamplingGroupLoopRegressionTests.cs @@ -490,6 +490,21 @@ public void ScheduleDoesNotRunAwayWhenSamplingIsSlowerThanTheInterval() Assert.That(restarts, Is.GreaterThan(0)); } + /// + /// Long waits convert to a delay without overflowing at the 1 GHz timestamp frequency + /// of Unix: the default one-hour sampling group would otherwise compute 3.6e19 ticks. + /// + [Test] + public void ScheduleWaitOfAnHourConvertsWithoutOverflowAtOneGigahertz() + { + const long frequency = 1_000_000_000; + var schedule = SamplingSchedule.Create(3_600_000, frequency, 0); + + TimeSpan wait = SamplingSchedule.ToTimeSpan(schedule.GetWait(0), frequency); + + Assert.That(wait, Is.EqualTo(TimeSpan.FromHours(1))); + } + private const long kTicksPerSecond = 10_000_000; private static long Ticks(double milliseconds) diff --git a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs index 7b0d53add2..0dac8f475d 100644 --- a/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs +++ b/tests/Opc.Ua.Server.Tests/ResourceManagerSnapshotTests.cs @@ -28,6 +28,7 @@ * ======================================================================*/ +using System; using System.Globalization; using System.Threading; using System.Threading.Tasks; @@ -73,7 +74,7 @@ public void UpdatesAndNewLocalesAreVisibleAfterASnapshotWasBuilt() /// written table, and see every translation once the writer is done. /// [Test] - public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() + public async Task ConcurrentReadersSeeACompleteSnapshotOnceWritesCompleteAsync() { using ResourceManager resourceManager = CreateResourceManager(); const int keyCount = 500; @@ -108,8 +109,8 @@ public void ConcurrentReadersSeeACompleteSnapshotOnceWritesComplete() "de-DE", "Wert" + ii.ToString(CultureInfo.InvariantCulture)); } - cts.Cancel(); - Assert.That(Task.WaitAll(readers, 30000), Is.True); + await cts.CancelAsync().ConfigureAwait(false); + await Task.WhenAll(readers).WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.That(wrongTexts, Is.Zero); for (int ii = 0; ii < keyCount; ii++) From 9bef80613bd9a67890cca0ea0cce153d9636f5db Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 11:55:52 +0200 Subject: [PATCH 17/19] perf: measure the client process in the load harness The harness already ran the server in a child process; the parent process is the client, so the same counters taken there measure the client stack without the server. - client-process and client-per-op lines (allocated bytes, CPU, GC, lock contention) for the measurement window; - --trace-side client records the harness itself with dotnet-trace; - --client managed drives ManagedSession with the V2 subscription engine, --callback item uses the classic per-item Notification event; - classic sessions no longer ask the server for diagnostics, as an application would not; - ab.sh takes SCENARIOS/ROUNDS/OUT, tolerates CRLF scenario files and passes defaults before the scenario so a scenario can override them; ab-summary.ps1 reports client and server medians separately. Co-Authored-By: Claude Opus 5.5 --- perf/README.md | 50 +++++-- perf/ServerLoadHarness/Program.cs | 216 ++++++++++++++++++++++++++---- perf/ab-summary.ps1 | 32 +++-- perf/ab.sh | 30 +++-- perf/client-scenarios.txt | 12 ++ perf/server-scenarios.txt | 6 + 6 files changed, 280 insertions(+), 66 deletions(-) create mode 100644 perf/client-scenarios.txt create mode 100644 perf/server-scenarios.txt diff --git a/perf/README.md b/perf/README.md index 5c1a70ae36..c86430d1a0 100644 --- a/perf/README.md +++ b/perf/README.md @@ -1,16 +1,18 @@ -# Server performance tooling +# Performance tooling -Tools for measuring server latency, throughput and GC cost under load. They are not part of -the product build (not in `UA.slnx`) and target `net10.0` only. +Tools for measuring server and client latency, throughput and GC cost under load. They are not +part of the product build (not in `UA.slnx`) and target `net10.0` only. | Path | Purpose | |---|---| -| `ServerLoadHarness/` | Starts a reference server in a child process and drives load against it from the parent process. | +| `ServerLoadHarness/` | Starts a reference server in a child process and drives load against it from the parent process, which is also the measured client. | | `TraceAnalyzer/` | Aggregates the `dotnet-trace` files the harness records (CPU, allocations, lock contention). | | `ab.sh`, `ab-summary.ps1` | Paired A/B runs of two harness builds, and the median summary. | +| `server-scenarios.txt`, `client-scenarios.txt` | The scenario lists used for the server and client comparisons. | The server runs in its own process, so the server-side numbers (CPU, allocated bytes, GC count -and pause time, `Monitor.LockContentionCount`) exclude the load generator. The NUnit +and pause time, `Monitor.LockContentionCount`) exclude the load generator, and the same +counters taken in the parent process measure the client stack without the server. The NUnit `LoadTest` fixture in `tests/Opc.Ua.Sessions.Tests` checks correctness under load; this harness measures it. @@ -35,21 +37,27 @@ perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe run --scenario | `--nodes` | 100 | Values per Read/Write request, or monitored items per subscription. | | `--inflight` | 1 | Outstanding requests per session (closed loop). | | `--security` | `none` | `none`, `sign` or `encrypt` (Basic256Sha256). | +| `--client` | `classic` | `classic`: `Session` with the classic subscription engine. `managed`: `ManagedSession` with the V2 subscription engine (`DefaultSubscriptionEngineFactory`). | +| `--callback` | `fast` | Classic data changes through `Subscription.FastDataChangeCallback` (`fast`) or the per-item `MonitoredItem.Notification` event (`item`). | | `--subs`, `--pub`, `--write-interval` | 5, 100, 100 | Subscriptions per session, publishing interval and writer interval in ms (`sub`). | | `--duration`, `--warmup` | 20, 10 | Measurement window and warm-up in seconds. | | `--diag`, `--audit` | 0 | Server diagnostics and auditing. | | `--trace ` | | Record the measurement window with `dotnet-trace`. | | `--trace-profile` | `cpu` | `cpu` (sampled thread time), `alloc` (allocation ticks), `contention` or `gc`. | +| `--trace-side` | `server` | The process to record: `server` (the child) or `client` (the harness itself). | | `--out ` | | Append the result block to a file. | Environment variables prefixed `SERVER_` are passed to the server process without the prefix, -for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes. +for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes. `DOTNET_gcServer=1` +set for the harness itself switches the client. -The result block reports client throughput and latency percentiles, the server deltas for the -window, and the server's allocated bytes and CPU time per request (or per notification). -Allocated bytes per operation are deterministic and the most reliable signal. On a shared -machine, throughput and latency vary from run to run, so compare builds with `ab.sh` rather -than with two separate runs. +The result block reports client throughput and latency percentiles, the deltas of both +processes for the window (`server:`, `client-process:`), and the allocated bytes and CPU time +per request (or per notification) of the server (`per-op:`) and of the client +(`client-per-op:`). Classic sessions are created without diagnostics in responses, as an +application would; the test fixture otherwise asks for them. Allocated bytes per operation are +deterministic and the most reliable signal. On a shared machine, throughput and latency vary +from run to run, so compare builds with `ab.sh` rather than with two separate runs. ## Analyse a trace @@ -62,15 +70,29 @@ dotnet perf/TraceAnalyzer/bin/Release/net10.0/TraceAnalyzer.dll contention trace The sampled thread-time profile also samples blocked threads. The `cpu` mode drops samples whose leaf frame is a known wait (thread-pool semaphore, I/O completion poll, GC suspension), so the remaining samples approximate on-CPU time. Spin-waiting in `Monitor.Enter_Slowpath` or -`Lock.TryEnterSlow` is real CPU and shows up there. +`Lock.TryEnterSlow` is real CPU and shows up there; check it against a `contention` trace, +though, because a thread blocked in the native part of `Monitor.Enter` has the same leaf frame. +Socket sends and receives on loopback show up as on-CPU kernel time. + +The `alloc` profile samples one allocation per ~100 KB and attributes the whole sample to the +object that crossed the threshold. That over-weights small objects allocated right after large +arrays, so read the type list as a pointer and the per-operation bytes in the result block as +the measurement. ## A/B comparison -Build the baseline harness in a second worktree, then alternate the two builds: +Build the baseline harness in a second worktree, then alternate the two builds. `SCENARIOS` +selects the scenario list (`server-scenarios.txt` by default, `client-scenarios.txt` for the +client), `ROUNDS` the rounds per scenario and `OUT` the result file: ```bash git worktree add --detach C:/uabase origin/master cp -r perf C:/uabase/ && (cd C:/uabase && CustomTestTarget=net10.0 dotnet build perf/ServerLoadHarness -c Release) -cd perf && BASELINE=C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe ./ab.sh +cd perf && BASELINE=C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe SCENARIOS=client-scenarios.txt ./ab.sh pwsh -File ab-summary.ps1 ``` + +The summary prints medians as +`baseline -> candidate (change)`: throughput, latency percentiles, client allocated bytes and CPU +per operation (`calloc`, `ccpu`), GC pause and lock contention in the window (`cpause`, +`ccont`), and the same for the server (`alloc`, `cpu`, `pause`, `cont`). diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index 320d3807ba..7d31d1befe 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -35,8 +35,9 @@ // run --scenario read|write|sub|browse [--sessions N] [--nodes K] [--inflight I] // [--duration S] [--warmup S] [--security none|sign|encrypt] [--diag 0|1] [--audit 0|1] // [--subs S] [--pub ms] [--write-interval ms] [--trace file] [--trace-profile p] [--out file] +// [--client classic|managed] [--callback fast|item] [--trace-side server|client] // Spawns the server child, drives the load and prints a report of client latency -// and server-side CPU / GC / allocation deltas for the measurement window. +// and the CPU / GC / allocation deltas of both processes for the measurement window. using System; using System.Collections; @@ -55,6 +56,7 @@ using Opc.Ua.Configuration; using Opc.Ua.Server.TestFramework; using Quickstarts.ReferenceServer; +using V2 = Opc.Ua.Client.Subscriptions; namespace Opc.Ua.Perf.ServerLoadHarness { @@ -193,7 +195,7 @@ private static async Task RunServerAsync(ApplicationInstance application, I await application.StartAsync(server).ConfigureAwait(false); Console.WriteLine(FormattableString.Invariant($"READY {endpointUrl} {Environment.ProcessId}")); - ServerStats mark = ServerStats.Capture(); + ProcessStats mark = ProcessStats.Capture(); string? line; while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) != null) { @@ -202,11 +204,11 @@ private static async Task RunServerAsync(ApplicationInstance application, I case "mark": GC.Collect(); GC.WaitForPendingFinalizers(); - mark = ServerStats.Capture(); + mark = ProcessStats.Capture(); Console.WriteLine("MARKED"); break; case "report": - Console.WriteLine("REPORT " + ServerStats.Capture().Delta(mark)); + Console.WriteLine("REPORT " + ProcessStats.Capture().Delta(mark)); break; case "quit": await server.StopAsync().ConfigureAwait(false); @@ -260,6 +262,9 @@ private static async Task RunDriverAsync(string[] args) await client .LoadClientConfigurationAsync(Path.Combine(Path.GetTempPath(), "uaperf-pki-client"), "PerfClient") .ConfigureAwait(false); + // The test fixture asks the server for diagnostics in every response; + // an application does not by default. + client.SessionFactory.ReturnDiagnostics = DiagnosticsMasks.None; string policy = security == "none" ? SecurityPolicies.None : SecurityPolicies.Basic256Sha256; MessageSecurityMode mode = security switch { @@ -277,18 +282,30 @@ await client } } + // classic: Session from the default session factory (classic subscription engine). + // managed: ManagedSession with the default V2 subscription engine. + string clientKind = Arg("client", "classic"); + ConfiguredEndpoint? managedEndpoint = clientKind == "managed" + ? await client.GetEndpointAsync(new Uri(url), policy, selected).ConfigureAwait(false) + : null; var sessions = new ISession[sessionCount]; var connectTime = Stopwatch.StartNew(); await Parallel.ForAsync( 0, sessionCount, new ParallelOptions { MaxDegreeOfParallelism = 32 }, - async (i, ct) => sessions[i] = await client - .ConnectAsync(new Uri(url), policy, selected) - .ConfigureAwait(false)) + async (i, ct) => sessions[i] = managedEndpoint != null + ? await new ManagedSessionBuilder(client.Config, telemetry) + .UseEndpoint(managedEndpoint) + .WithSessionTimeout(TimeSpan.FromSeconds(60)) + .ConnectAsync(ct) + .ConfigureAwait(false) + : await client + .ConnectAsync(new Uri(url), policy, selected) + .ConfigureAwait(false)) .ConfigureAwait(false); Console.WriteLine(FormattableString.Invariant( - $"connected {sessionCount} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); + $"connected {sessionCount} {clientKind} sessions ({security}) in {connectTime.ElapsedMilliseconds} ms")); Workload workload = scenario == "sub" ? new SubscribeWorkload( @@ -296,7 +313,8 @@ await Parallel.ForAsync( ArgInt("nodes", 100), ArgInt("subs", 5), ArgInt("pub", 100), - ArgInt("write-interval", 100)) + ArgInt("write-interval", 100), + Arg("callback", "fast") == "item") : new RequestWorkload(scenario, sessions, ArgInt("nodes", 100), ArgInt("inflight", 1)); await workload.SetupAsync().ConfigureAwait(false); @@ -304,7 +322,11 @@ await Parallel.ForAsync( Task run = workload.RunAsync(stop.Token); await Task.Delay(TimeSpan.FromSeconds(warmupSeconds)).ConfigureAwait(false); - using Process? tracer = StartTracer(server.Id, durationSeconds, out string? trace); + bool traceClient = Arg("trace-side", "server") == "client"; + using Process? tracer = StartTracer( + traceClient ? Environment.ProcessId : server.Id, + durationSeconds, + out string? trace); if (tracer != null) { await Task.Delay(2000).ConfigureAwait(false); @@ -312,9 +334,13 @@ await Parallel.ForAsync( await server.StandardInput.WriteLineAsync("mark").ConfigureAwait(false); Take(serverLines, "MARKED"); + GC.Collect(); + GC.WaitForPendingFinalizers(); + ProcessStats clientMark = ProcessStats.Capture(); workload.ResetStats(); var window = Stopwatch.StartNew(); await Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ConfigureAwait(false); + string clientProcess = ProcessStats.Capture().Delta(clientMark); await server.StandardInput.WriteLineAsync("report").ConfigureAwait(false); string report = Take(serverLines, "REPORT")[7..]; string clientReport = workload.Report(window.Elapsed.TotalSeconds); @@ -339,14 +365,19 @@ await Parallel.ForAsync( .Append(string.Join(' ', args.Skip(1))) .Append('\n') .Append(" client: ").Append(clientReport).Append('\n') - .Append(" server: ").Append(report).Append('\n'); + .Append(" server: ").Append(report).Append('\n') + .Append(" client-process: ").Append(clientProcess).Append('\n'); double operations = workload.Operations; if (operations > 0) { - (double allocatedMb, double cpuMs) = ParseServerReport(report); + (double allocatedMb, double cpuMs) = ParseProcessReport(report); result.Append( CultureInfo.InvariantCulture, $" per-op: allocB={allocatedMb * 1048576 / operations:F0} cpuUs={cpuMs * 1000 / operations:F1} ({workload.OperationName})\n"); + (allocatedMb, cpuMs) = ParseProcessReport(clientProcess); + result.Append( + CultureInfo.InvariantCulture, + $" client-per-op: allocB={allocatedMb * 1048576 / operations:F0} cpuUs={cpuMs * 1000 / operations:F1} ({workload.OperationName})\n"); } Console.Write(result.ToString()); if (s_args.TryGetValue("out", out string? outFile)) @@ -432,7 +463,7 @@ private static string Take(BlockingCollection lines, string prefix) } } - private static (double AllocatedMb, double CpuMs) ParseServerReport(string report) + private static (double AllocatedMb, double CpuMs) ParseProcessReport(string report) { Dictionary values = report .Split(' ') @@ -448,7 +479,7 @@ private static (double AllocatedMb, double CpuMs) ParseServerReport(string repor /// /// Process-wide counters of the server process, captured at the start and end of the window. /// - internal sealed record ServerStats( + internal sealed record ProcessStats( long Allocated, int Gen0, int Gen1, @@ -461,10 +492,10 @@ internal sealed record ServerStats( long WorkingSet, long Heap) { - public static ServerStats Capture() + public static ProcessStats Capture() { using var process = Process.GetCurrentProcess(); - return new ServerStats( + return new ProcessStats( GC.GetTotalAllocatedBytes(true), GC.CollectionCount(0), GC.CollectionCount(1), @@ -478,7 +509,7 @@ public static ServerStats Capture() GC.GetGCMemoryInfo().HeapSizeBytes); } - public string Delta(ServerStats mark) + public string Delta(ProcessStats mark) { double seconds = Stopwatch.GetElapsedTime(mark.Timestamp, Timestamp).TotalSeconds; TimeSpan cpu = Cpu - mark.Cpu; @@ -680,13 +711,21 @@ private async Task LoopAsync(ISession session, CancellationToken ct) /// internal sealed class SubscribeWorkload : Workload { - public SubscribeWorkload(ISession[] sessions, int nodes, int subscriptions, int publishingInterval, int writeInterval) + public SubscribeWorkload( + ISession[] sessions, + int nodes, + int subscriptions, + int publishingInterval, + int writeInterval, + bool itemCallback) { m_sessions = sessions; m_nodes = nodes; m_subscriptions = subscriptions; m_publishingInterval = publishingInterval; m_writeInterval = writeInterval; + m_itemCallback = itemCallback; + m_handler = new NotificationHandler(this); } public override double Operations => Interlocked.Read(ref m_notifications); @@ -754,27 +793,40 @@ public override string Report(double seconds) private async Task CreateSubscriptionAsync(ISession session, CancellationToken ct) { + if (session is ManagedSession managed) + { + await CreateManagedSubscriptionAsync(managed, ct).ConfigureAwait(false); + return; + } #pragma warning disable CA2000 // ownership transfers to the session in AddSubscription; disposed below on failure before that var subscription = new Subscription(session.DefaultSubscription) { PublishingInterval = m_publishingInterval, KeepAliveCount = 10, - LifetimeCount = 100, - FastDataChangeCallback = OnDataChange + LifetimeCount = 100 }; #pragma warning restore CA2000 + if (!m_itemCallback) + { + subscription.FastDataChangeCallback = OnDataChange; + } try { foreach (NodeId nodeId in m_nodeIds) { - subscription.AddItem(new MonitoredItem(subscription.DefaultItem) + var item = new MonitoredItem(subscription.DefaultItem) { StartNodeId = nodeId, AttributeId = Attributes.Value, MonitoringMode = MonitoringMode.Reporting, SamplingInterval = 0, QueueSize = 1 - }); + }; + if (m_itemCallback) + { + item.Notification += OnItemNotification; + } + subscription.AddItem(item); } // the session owns and disposes the subscription from here on session.AddSubscription(subscription); @@ -793,17 +845,121 @@ private void OnDataChange(Subscription subscription, DataChangeNotification noti int count = 0; foreach (MonitoredItemNotification item in notification.MonitoredItems) { - if (item.Value.WrappedValue.TryGetValue(out int counter) && counter > 0) + RecordValue(now, item.Value); + count++; + } + Interlocked.Add(ref m_notifications, count); + } + + private void OnItemNotification(MonitoredItem monitoredItem, MonitoredItemNotificationEventArgs e) + { + if (e.NotificationValue is MonitoredItemNotification item) + { + RecordValue(Stopwatch.GetTimestamp(), item.Value); + Interlocked.Increment(ref m_notifications); + } + } + + private void RecordValue(long now, DataValue value) + { + if (value.WrappedValue.TryGetValue(out int counter) && counter > 0) + { + long written = Volatile.Read(ref m_writeTicks[counter & 0xFFFF]); + if (written != 0) { - long written = Volatile.Read(ref m_writeTicks[counter & 0xFFFF]); - if (written != 0) + m_latency.Record(now - written); + } + } + } + + private async Task CreateManagedSubscriptionAsync(ManagedSession session, CancellationToken ct) + { + V2.ISubscription subscription = session.AddSubscription( + m_handler, + new V2.SubscriptionOptions + { + PublishingInterval = TimeSpan.FromMilliseconds(m_publishingInterval), + PublishingEnabled = true, + KeepAliveCount = 10, + LifetimeCount = 100 + }); + for (int i = 0; i < m_nodeIds.Count; i++) + { + subscription.TryAddMonitoredItem( + i.ToString(CultureInfo.InvariantCulture), + new V2.MonitoredItems.MonitoredItemOptions { - m_latency.Record(now - written); - } + StartNodeId = m_nodeIds[i], + AttributeId = Attributes.Value, + MonitoringMode = MonitoringMode.Reporting, + SamplingInterval = TimeSpan.Zero, + QueueSize = 1 + }, + out _); + } + while (!subscription.Created || subscription.MonitoredItems.Items.Any(item => !item.Created)) + { + await Task.Delay(50, ct).ConfigureAwait(false); + } + } + + /// + /// V2 engine handler; data changes are counted like the classic fast callback. + /// + private sealed class NotificationHandler : V2.ISubscriptionNotificationHandler + { + public NotificationHandler(SubscribeWorkload owner) + { + m_owner = owner; + } + + public ValueTask OnDataChangeNotificationAsync( + V2.ISubscription subscription, + uint sequenceNumber, + DateTime publishTime, + ReadOnlyMemory notification, + V2.PublishState publishStateMask, + IReadOnlyList stringTable) + { + long now = Stopwatch.GetTimestamp(); + foreach (V2.DataValueChange change in notification.Span) + { + m_owner.RecordValue(now, change.Value); } - count++; + Interlocked.Add(ref m_owner.m_notifications, notification.Length); + return ValueTask.CompletedTask; } - Interlocked.Add(ref m_notifications, count); + + public ValueTask OnEventDataNotificationAsync( + V2.ISubscription subscription, + uint sequenceNumber, + DateTime publishTime, + ReadOnlyMemory notification, + V2.PublishState publishStateMask, + IReadOnlyList stringTable) + { + return ValueTask.CompletedTask; + } + + public ValueTask OnKeepAliveNotificationAsync( + V2.ISubscription subscription, + uint sequenceNumber, + DateTime publishTime, + V2.PublishState publishStateMask) + { + return ValueTask.CompletedTask; + } + + public ValueTask OnSubscriptionStateChangedAsync( + V2.ISubscription subscription, + V2.SubscriptionState state, + V2.PublishState publishStateMask, + CancellationToken ct = default) + { + return ValueTask.CompletedTask; + } + + private readonly SubscribeWorkload m_owner; } private readonly ISession[] m_sessions; @@ -811,6 +967,8 @@ private void OnDataChange(Subscription subscription, DataChangeNotification noti private readonly int m_subscriptions; private readonly int m_publishingInterval; private readonly int m_writeInterval; + private readonly bool m_itemCallback; + private readonly NotificationHandler m_handler; private readonly long[] m_writeTicks = new long[1 << 16]; private readonly LatencyRecorder m_latency = new(); private long m_notifications; diff --git a/perf/ab-summary.ps1 b/perf/ab-summary.ps1 index 6f2ca38803..0ba48c8ada 100644 --- a/perf/ab-summary.ps1 +++ b/perf/ab-summary.ps1 @@ -1,22 +1,36 @@ param($file = "ab-results.txt") +# Medians per scenario and side of the result blocks that ab.sh appends. +# alloc/cpu/pause/cont are server-side, calloc/ccpu/cpause/ccont client-side. $rows = @() $cur = $null foreach ($l in Get-Content $file) { - if ($l -match '^### side=(\w) round=(\d) (.*)$') { $cur = @{ side = $Matches[1]; scen = $Matches[3].Trim() }; continue } + if ($l -match '^### side=(\w) round=(\d+) (.*)$') { $cur = @{ side = $Matches[1]; scen = $Matches[3].Trim() }; continue } if ($null -eq $cur) { continue } - if ($l -match 'req/s=(\d+)') { $cur.tput = [double]$Matches[1] } - if ($l -match 'notif/s=(\d+)') { $cur.tput = [double]$Matches[1] } - if ($l -match 'p50=([\d.]+)ms p90=([\d.]+)ms p99=([\d.]+)ms') { $cur.p50 = [double]$Matches[1]; $cur.p90 = [double]$Matches[2]; $cur.p99 = [double]$Matches[3] } - if ($l -match 'pauseMs=(\d+)') { $cur.pause = [double]$Matches[1] } - if ($l -match 'contention=(\d+)') { $cur.cont = [double]$Matches[1] } - if ($l -match 'per-op: allocB=(\d+) cpuUs=([\d.]+)') { $cur.alloc = [double]$Matches[1]; $cur.cpu = [double]$Matches[2]; $rows += [pscustomobject]$cur; $cur = $null } + if ($l -match '^\s+client: ') { + if ($l -match 'req/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'notif/s=(\d+)') { $cur.tput = [double]$Matches[1] } + if ($l -match 'p50=([\d.]+)ms p90=([\d.]+)ms p99=([\d.]+)ms') { $cur.p50 = [double]$Matches[1]; $cur.p90 = [double]$Matches[2]; $cur.p99 = [double]$Matches[3] } + } + elseif ($l -match '^\s+server: ') { + if ($l -match 'pauseMs=(\d+)') { $cur.pause = [double]$Matches[1] } + if ($l -match 'contention=(\d+)') { $cur.cont = [double]$Matches[1] } + } + elseif ($l -match '^\s+client-process: ') { + if ($l -match 'pauseMs=(\d+)') { $cur.cpause = [double]$Matches[1] } + if ($l -match 'contention=(\d+)') { $cur.ccont = [double]$Matches[1] } + } + elseif ($l -match '^\s+per-op: allocB=(\d+) cpuUs=([\d.]+)') { $cur.alloc = [double]$Matches[1]; $cur.cpu = [double]$Matches[2] } + elseif ($l -match '^\s+client-per-op: allocB=(\d+) cpuUs=([\d.]+)') { + $cur.calloc = [double]$Matches[1]; $cur.ccpu = [double]$Matches[2] + $rows += [pscustomobject]$cur; $cur = $null + } } -function Med($xs) { $s = @($xs | Sort-Object); if ($s.Count -eq 0) { return [double]::NaN }; return $s[[int][Math]::Floor($s.Count / 2)] } +function Med($xs) { $s = @($xs | Where-Object { $null -ne $_ } | Sort-Object); if ($s.Count -eq 0) { return [double]::NaN }; return $s[[int][Math]::Floor($s.Count / 2)] } $out = @() foreach ($g in ($rows | Group-Object scen)) { $a = $g.Group | Where-Object side -eq 'A'; $b = $g.Group | Where-Object side -eq 'B' $o = [ordered]@{ scenario = $g.Name; n = "$(@($a).Count)/$(@($b).Count)" } - foreach ($m in 'tput', 'p50', 'p90', 'p99', 'alloc', 'cpu', 'pause', 'cont') { + foreach ($m in 'tput', 'p50', 'p90', 'p99', 'calloc', 'ccpu', 'cpause', 'ccont', 'alloc', 'cpu', 'pause', 'cont') { $ma = Med ($a | ForEach-Object { $_.$m }); $mb = Med ($b | ForEach-Object { $_.$m }) $o["$m"] = "{0:G4} -> {1:G4} ({2:+0;-0}%)" -f $ma, $mb, (100 * ($mb - $ma) / $ma) } diff --git a/perf/ab.sh b/perf/ab.sh index 3f26ae6dcd..aa8c4e5984 100644 --- a/perf/ab.sh +++ b/perf/ab.sh @@ -1,22 +1,24 @@ #!/bin/bash +# Paired A/B runs: alternates a baseline and a candidate harness build per scenario and round. # BASELINE: harness built in a second worktree of the commit to compare against. +# SCENARIOS: file with one scenario per line (default: server-scenarios.txt). +# ROUNDS: rounds per scenario (default 3). OUT: result file (default ab-results.txt). A=${BASELINE:-C:/uabase/perf/ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe} B=./ServerLoadHarness/bin/Release/net10.0/ServerLoadHarness.exe -O=ab-results.txt; rm -f $O -while IFS= read -r s; do - for r in 1 2 3; do +O=${OUT:-ab-results.txt}; rm -f "$O" +ROUNDS=${ROUNDS:-3} +SCENARIOS=${SCENARIOS:-server-scenarios.txt} +while IFS= read -r s || [ -n "$s" ]; do + # tolerate CRLF scenario files + s=${s%$'\r'} + [ -z "$s" ] && continue + for r in $(seq 1 "$ROUNDS"); do for side in A B; do exe=$A; [ $side = B ] && exe=$B - echo "### side=$side round=$r $s" >> $O - timeout 300 $exe run $s --sessions 20 --duration 12 --warmup 6 --out $O >/dev/null 2>&1 + echo "### side=$side round=$r $s" >> "$O" + # defaults first, so that a scenario can override them + timeout 300 $exe run --sessions 20 --duration 12 --warmup 6 $s --out "$O" >/dev/null 2>&1 done done -done <<'S' ---scenario read --nodes 1 ---scenario read --nodes 100 ---scenario write --nodes 100 ---scenario browse ---scenario read --nodes 100 --security encrypt ---scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 -S -echo done >> $O +done < "$SCENARIOS" +echo done >> "$O" diff --git a/perf/client-scenarios.txt b/perf/client-scenarios.txt new file mode 100644 index 0000000000..327787d1cb --- /dev/null +++ b/perf/client-scenarios.txt @@ -0,0 +1,12 @@ +--scenario read --nodes 1 +--scenario read --nodes 100 +--scenario write --nodes 100 +--scenario browse +--scenario read --nodes 1 --security encrypt +--scenario read --nodes 100 --security encrypt +--scenario read --nodes 1 --client managed +--scenario read --nodes 1 --sessions 200 +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 --callback item +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 --client managed +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 --security encrypt diff --git a/perf/server-scenarios.txt b/perf/server-scenarios.txt new file mode 100644 index 0000000000..966cab3f10 --- /dev/null +++ b/perf/server-scenarios.txt @@ -0,0 +1,6 @@ +--scenario read --nodes 1 +--scenario read --nodes 100 +--scenario write --nodes 100 +--scenario browse +--scenario read --nodes 100 --security encrypt +--scenario sub --subs 5 --nodes 100 --pub 100 --write-interval 50 From a40db09cfe069b91c811e0c7bb5e3faa10b94128 Mon Sep 17 00:00:00 2001 From: Roman Ettlinger Date: Fri, 9 Oct 2026 11:55:52 +0200 Subject: [PATCH 18/19] Client: cut per-request allocation and CPU in the channel and session Profiling the client process under load (paired A/B against the server branch, 20 sessions, workstation GC): Read 1 value +21% req/s with client alloc/request -46% and CPU/request -13%; 200 sessions +29%; SignAndEncrypt Read 1 value +28% (CPU -24%); Write 100 values client alloc -63%. The server shares the channel code and allocates 20-26% less per small request. - ActivitySource cached per instance in Session, SessionClientBatched, the classic Subscription and engine and the TCP/HTTPS client channels: StartActivity walked the stack (GetCallingAssembly) on every call. - Channel writes: an ordered queue drained by one thread-pool writer (local queue of the issuing thread) replaces the per-write ContinueWith/Unwrap chain and Task.Run; successful writes are reported inline, and the client channel only enters its gate for a failed write. - TcpByteTransport: no linked CancellationTokenSource for a non-cancellable token, single-chunk messages use the memory send, pooled async builders on .NET 6+, no closure per vectored send. - BinaryEncoder.GetNodeIdEncoding boxed every NodeId through Convert.ToUInt32(object) and Convert.ToByte(enum). - Symmetric chunk headers are written and read directly instead of through an encoder/decoder, stream and reader/writer per chunk; a short chunk still reports the decoder's error. - AES-CBC: one Aes per channel token and direction, with the one-shot EncryptCbc/DecryptCbc on .NET 6+, instead of a new instance (key import) per message. - ChannelAsyncOperation: a task-free completion wait for requests, cancellation by registration instead of WaitAsync, a shared timer callback and a shared box for default results. - Session keep alive: a response records the time instead of moving the timer (timer queue lock); the timer defers itself to one interval after the last response. Tests cover the write queue (order, one send at a time, off the caller, failures), the completion wait (cancellation, disposal, races), the AES cache, the transport send paths and the keep-alive deferral. Co-Authored-By: Claude Opus 5.5 --- src/Opc.Ua.Client/Session/Session.cs | 122 ++++-- .../Session/SessionClientBatched.cs | 46 ++- .../Subscription/ClassicSubscriptionEngine.cs | 24 +- .../Subscription/Classic/Subscription.cs | 58 ++- .../Security/Certificates/CryptoUtils.cs | 198 ++++++++-- .../Stack/Https/HttpsTransportChannel.cs | 11 +- .../Stack/Tcp/ChannelAsyncOperation.cs | 166 ++++++++- src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs | 16 + src/Opc.Ua.Core/Stack/Tcp/TcpByteTransport.cs | 127 +++++-- .../Stack/Tcp/UaSCBinaryChannel.Symmetric.cs | 80 ++-- .../Stack/Tcp/UaSCBinaryChannel.cs | 176 +++++++-- .../Stack/Tcp/UaSCBinaryClientChannel.cs | 23 +- src/Opc.Ua.Types/Encoders/BinaryEncoder.cs | 4 +- .../Session/SessionKeepAliveTests.cs | 69 +++- .../Certificates/CryptoUtilsAesCacheTests.cs | 231 ++++++++++++ .../ChannelAsyncOperationWaitTests.cs | 254 +++++++++++++ .../Stack/Transport/TcpByteTransportTests.cs | 113 ++++++ .../UaSCBinaryChannelWriteQueueTests.cs | 349 ++++++++++++++++++ 18 files changed, 1863 insertions(+), 204 deletions(-) create mode 100644 tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs create mode 100644 tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs create mode 100644 tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs diff --git a/src/Opc.Ua.Client/Session/Session.cs b/src/Opc.Ua.Client/Session/Session.cs index 51c2be50ac..df781717b3 100644 --- a/src/Opc.Ua.Client/Session/Session.cs +++ b/src/Opc.Ua.Client/Session/Session.cs @@ -348,7 +348,7 @@ private Session( // Create timer for keep alive event triggering but in off state m_keepAliveTimer = m_timeProvider.CreateTimer( - _ => m_keepAliveEvent.Set(), + _ => OnKeepAliveTimer(), this, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); @@ -1080,7 +1080,7 @@ public event RenewUserIdentityEventHandler RenewUserIdentity /// public virtual void Snapshot(out SessionState state) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); Snapshot(out SessionConfiguration configuration); // Snapshot subscription state @@ -1102,7 +1102,7 @@ public virtual void Snapshot(out SessionState state) /// public virtual void Restore(SessionState state) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ThrowIfDisposed(); RestoreSessionState(state); if (state.Subscriptions.IsEmpty) @@ -1246,7 +1246,7 @@ public virtual void Save( IEnumerable subscriptions, IEnumerable? knownTypes = null) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); // Snapshot subscription state var subscriptionStates = new List(); foreach (Subscription subscription in subscriptions) @@ -1276,7 +1276,7 @@ public virtual IEnumerable Load( bool transferSubscriptions = false, IEnumerable? knownTypes = null) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); IServiceMessageContext context = MessageContext ?? throw new InvalidOperationException("Missing service message context"); @@ -1392,7 +1392,7 @@ private async Task OpenCoreAsync( ArrayOf refreshedEndpoints = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ArrayOf discoveryServerEndpoints = refreshedEndpoints.IsEmpty ? m_endpoint.DiscoveryEndpoints @@ -1830,7 +1830,7 @@ public async ValueTask UpdateIdentityAsync( } ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); await AcquireIdentityUpdateLockAsync(ct).ConfigureAwait(false); CancellationTokenSource lockHolder = EnterReconnectLockHolder(ct); @@ -2303,7 +2303,7 @@ public async Task UpdateSessionAsync( CancellationToken ct = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ByteString serverNonce = default; lock (m_lock) @@ -2526,7 +2526,7 @@ public async Task RemoveSubscriptionAsync( CancellationToken ct = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); if (subscription == null) { throw new ArgumentNullException(nameof(subscription)); @@ -2558,7 +2558,7 @@ public async Task RemoveSubscriptionsAsync( CancellationToken ct = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); if (subscriptions == null) { throw new ArgumentNullException(nameof(subscriptions)); @@ -2588,7 +2588,7 @@ public async Task ReactivateSubscriptionsAsync( CancellationToken ct = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ArrayOf subscriptionIds = CreateSubscriptionIdsForTransfer(subscriptions); int failedSubscriptions = 0; @@ -2702,7 +2702,7 @@ private async Task TransferSubscriptionsCoreAsync( ICollection? notTransferred, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ArrayOf subscriptionIds = CreateSubscriptionIdsForTransfer( subscriptions, sessionRecreatedInPlace); @@ -2874,7 +2874,7 @@ public Task FetchNamespaceTablesAsync(CancellationToken ct = default) private async Task FetchNamespaceTablesAsync(ISessionClient serviceClient, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ArrayOf nodesToRead = PrepareNamespaceTableNodesToRead(); // read from server. @@ -2899,7 +2899,7 @@ private async Task FetchNamespaceTablesAsync(ISessionClient serviceClient, Cance /// public async Task FetchTypeTreeAsync(ExpandedNodeId typeId, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); await FetchTypeTreeAsync(typeId, [typeId], ct) .ConfigureAwait(false); } @@ -2909,7 +2909,7 @@ public async Task FetchTypeTreeAsync( ArrayOf typeIds, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); var visited = new HashSet(); foreach (ExpandedNodeId typeId in typeIds) { @@ -2986,7 +2986,7 @@ public Task FetchOperationLimitsAsync(CancellationToken ct) private async Task FetchOperationLimitsAsync(SessionClient serviceClient, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); // Helper extraction static uint GetUInt32(ref int index, ArrayOf values, ArrayOf errors) @@ -3512,7 +3512,7 @@ private async Task RecreateInPlaceCoreAsync( bool bindSuppliedChannel = false) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); NodeId previousSessionId = SessionId; IManagedTransportChannel? oldManagedLease = ManagedChannel; @@ -4024,7 +4024,7 @@ public virtual async Task CloseAsync( Closing = true; - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); try { // stop the keep alive timer. @@ -4123,7 +4123,7 @@ await WaitForOrCancelOutstandingPublishRequestsAsync(timeoutCts.Token) public async Task ReloadInstanceCertificateAsync(CancellationToken ct = default) { ThrowIfDisposed(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); await AcquireIdentityUpdateLockAsync(ct).ConfigureAwait(false); try { @@ -4233,7 +4233,7 @@ await RecreateInPlaceCoreAsync( return; } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); bool resetReconnect = false; try { @@ -4557,7 +4557,7 @@ await identityToken.EncryptAsync( uint sequenceNumber, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); // Republish is part of the classic publish flow. With the V2 // subscription engine, the SubscriptionManager handles republish @@ -4634,7 +4634,7 @@ private async Task RecreateSubscriptionsAsync( CancellationToken ct, bool sessionRecreatedInPlace = false) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); bool transferred = false; // Per-subscription outcome of the transfer. Only an in-place // recreate needs it: a subscription the server did take over is @@ -4778,7 +4778,7 @@ protected virtual void OnApplicationCertificateError( /// /// Starts a timer to check that the connection to the server is still available. /// - private async ValueTask StartKeepAliveTimerAsync() + internal async ValueTask StartKeepAliveTimerAsync() { int keepAliveInterval = m_keepAliveInterval; @@ -4817,7 +4817,8 @@ private async ValueTask StartKeepAliveTimerAsync() m_keepAliveCancellation = keepAliveCancellation; } - // send initial keep alive. + // send initial keep alive, not deferred by earlier responses. + Volatile.Write(ref m_keepAliveDeferredFrom, 0); m_keepAliveTimer.Change(TimeSpan.Zero, TimeSpan.FromMilliseconds(m_keepAliveInterval)); } } @@ -4885,7 +4886,7 @@ protected override void RequestCompleted( UpdateLastKeepAliveTime(); } - ResetKeepAliveTimer(); + DeferKeepAlive(); } base.RequestCompleted(request, response!, serviceName); @@ -4905,6 +4906,54 @@ private void UpdateLastKeepAliveTime() Interlocked.Exchange(ref m_lastKeepAliveTimestamp, m_timeProvider.GetTimestamp()); } + /// + /// Defers the next keep alive to one keep alive interval from now. + /// + /// + /// Called for every successful response, so it only records the time + /// instead of moving the timer, which takes the timer queue lock. The + /// timer callback moves the timer when it fires before the deferred time. + /// + private void DeferKeepAlive() + { + Volatile.Write(ref m_keepAliveDeferredFrom, m_timeProvider.GetTimestamp()); + } + + /// + /// Signals the keep alive worker unless a response arrived within the + /// last keep alive interval. Then the timer is moved to one interval after + /// that response, which is when moving it on every response would have + /// made it fire. + /// + private void OnKeepAliveTimer() + { + long deferredFrom = Volatile.Read(ref m_keepAliveDeferredFrom); + if (deferredFrom != 0) + { + var interval = TimeSpan.FromMilliseconds(m_keepAliveInterval); + TimeSpan remaining = interval - m_timeProvider.GetElapsedTime(deferredFrom); + if (remaining > TimeSpan.Zero) + { + lock (m_lock) + { + if (m_keepAliveWorker != null && !Disposed) + { + try + { + m_keepAliveTimer.Change(remaining, interval); + } + catch (ObjectDisposedException) + { + // the session is being disposed + } + } + } + return; + } + } + m_keepAliveEvent.Set(); + } + /// /// Reset the timer used to send keep alive messages. /// @@ -6706,6 +6755,28 @@ protected virtual void ProcessResponseAdditionalHeader( protected CertificateEntry? m_instanceCertificateEntry; #pragma warning restore CA2213 + /// + /// The activity source for service calls, resolved once per telemetry + /// context: the GetActivitySource extension walks the stack to find the + /// calling assembly, which is too expensive to repeat for every call. + /// + private ActivitySource CachedActivitySource + { + get + { + ITelemetryContext telemetry = m_telemetry; + Tuple? cached = m_activitySource; + if (cached == null || !ReferenceEquals(cached.Item1, telemetry)) + { + cached = Tuple.Create(telemetry, telemetry.GetActivitySource()); + m_activitySource = cached; + } + return cached.Item2; + } + } + + private Tuple? m_activitySource; + /// /// The session telemetry context /// @@ -6859,6 +6930,7 @@ public BackgroundWorkScope(Session owner) #pragma warning restore CA2213 private long m_lastKeepAliveTime; private long m_lastKeepAliveTimestamp; + private long m_keepAliveDeferredFrom; private StatusCode m_lastKeepAliveErrorStatusCode; private ServerState m_serverState; private int m_keepAliveInterval; diff --git a/src/Opc.Ua.Client/Session/SessionClientBatched.cs b/src/Opc.Ua.Client/Session/SessionClientBatched.cs index 5bc9402894..72355c2624 100644 --- a/src/Opc.Ua.Client/Session/SessionClientBatched.cs +++ b/src/Opc.Ua.Client/Session/SessionClientBatched.cs @@ -99,7 +99,7 @@ async ValueTask BrowseBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); BrowseResponse? response = null; InitResponseCollections( @@ -190,7 +190,7 @@ async ValueTask BrowseNextBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); BrowseNextResponse? response = null; InitResponseCollections( @@ -263,7 +263,7 @@ async ValueTask TranslateBrowsePathsToNod uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); TranslateBrowsePathsToNodeIdsResponse? response = null; InitResponseCollections( @@ -324,7 +324,7 @@ async ValueTask RegisterNodesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); RegisterNodesResponse? response = null; var registeredNodeIds = new List(); @@ -369,7 +369,7 @@ async ValueTask UnregisterNodesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); UnregisterNodesResponse? response = null; foreach (ArrayOf batchNodesToUnregister in nodesToUnregister @@ -420,7 +420,7 @@ async ValueTask ReadBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); ReadResponse? response = null; InitResponseCollections( @@ -507,7 +507,7 @@ async ValueTask HistoryReadBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); HistoryReadResponse? response = null; InitResponseCollections( @@ -587,7 +587,7 @@ async ValueTask WriteBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); WriteResponse? response = null; InitResponseCollections( @@ -658,7 +658,7 @@ async ValueTask HistoryUpdateBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); HistoryUpdateResponse? response = null; InitResponseCollections( @@ -721,7 +721,7 @@ async ValueTask CallBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); CallResponse? response = null; InitResponseCollections( @@ -797,7 +797,7 @@ async ValueTask CreateMonitoredItemsBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); CreateMonitoredItemsResponse? response = null; InitResponseCollections( @@ -883,7 +883,7 @@ async ValueTask ModifyMonitoredItemsBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); ModifyMonitoredItemsResponse? response = null; InitResponseCollections( @@ -960,7 +960,7 @@ async ValueTask SetMonitoringModeBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); SetMonitoringModeResponse? response = null; InitResponseCollections( @@ -1042,7 +1042,7 @@ async ValueTask SetTriggeringBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); SetTriggeringResponse? response = null; InitResponseCollections( @@ -1198,7 +1198,7 @@ async ValueTask DeleteMonitoredItemsBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); DeleteMonitoredItemsResponse? response = null; InitResponseCollections( @@ -1262,7 +1262,7 @@ async ValueTask AddNodesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); AddNodesResponse? response = null; InitResponseCollections( @@ -1322,7 +1322,7 @@ async ValueTask AddReferencesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); AddReferencesResponse? response = null; InitResponseCollections( @@ -1385,7 +1385,7 @@ async ValueTask DeleteNodesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); DeleteNodesResponse? response = null; InitResponseCollections( @@ -1448,7 +1448,7 @@ async ValueTask DeleteReferencesBatchedAsync( uint operationLimit, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); requestHeader ??= new RequestHeader(); DeleteReferencesResponse? response = null; InitResponseCollections( @@ -1736,6 +1736,14 @@ static void UpdateDiagnosticInfoIndexes(DiagnosticInfo? diagnosticInfo, } } + /// + /// The activity source for service calls, resolved once: the + /// GetActivitySource extension walks the stack to find the calling + /// assembly, which is too expensive to repeat for every call. + /// + private ActivitySource CachedActivitySource => m_activitySource ??= m_telemetry.GetActivitySource(); + + private ActivitySource? m_activitySource; private readonly ITelemetryContext m_telemetry; } } diff --git a/src/Opc.Ua.Client/Session/Subscription/ClassicSubscriptionEngine.cs b/src/Opc.Ua.Client/Session/Subscription/ClassicSubscriptionEngine.cs index 2ae492acd9..b6b2acafb5 100644 --- a/src/Opc.Ua.Client/Session/Subscription/ClassicSubscriptionEngine.cs +++ b/src/Opc.Ua.Client/Session/Subscription/ClassicSubscriptionEngine.cs @@ -341,8 +341,7 @@ private bool BeginPublishCore(int timeout) try { - Activity? activity = m_context.Telemetry - .StartActivity(); + Activity? activity = CachedActivitySource.StartActivity(); Task task = m_context.PublishAsync( requestHeader, acknowledgementsToSend, @@ -1198,6 +1197,26 @@ internal static bool IsWithinOutOfOrderThreshold(uint sequenceNumber, uint lates private const int kDefaultPublishRequestCount = 1; private const int kPublishRequestSequenceNumberOutOfOrderThreshold = 10; private const int kPublishRequestSequenceNumberOutdatedThreshold = 100; + + /// + /// The activity source for publish requests, resolved once per telemetry + /// context: the GetActivitySource extension walks the stack to find the + /// calling assembly, which is too expensive to repeat for every request. + /// + private ActivitySource CachedActivitySource + { + get + { + ITelemetryContext telemetry = m_context.Telemetry; + Tuple? cached = m_activitySource; + if (cached == null || !ReferenceEquals(cached.Item1, telemetry)) + { + cached = Tuple.Create(telemetry, telemetry.GetActivitySource()); + m_activitySource = cached; + } + return cached.Item2; + } + } private readonly ISubscriptionEngineContext m_context; private readonly ILogger m_logger; private readonly ILogger m_eventLogger; @@ -1205,6 +1224,7 @@ internal static bool IsWithinOutOfOrderThreshold(uint sequenceNumber, uint lates private readonly BackgroundTaskScope m_backgroundWork; private readonly Lock m_acknowledgementsToSendLock = new(); private List m_acknowledgementsToSend = []; + private Tuple? m_activitySource; private int m_unrecordedPublishRequests; private int m_tooManyPublishRequests; private int m_minPublishRequestCount; diff --git a/src/Opc.Ua.Client/Subscription/Classic/Subscription.cs b/src/Opc.Ua.Client/Subscription/Classic/Subscription.cs index 537b0f0d0e..1bd34b6591 100644 --- a/src/Opc.Ua.Client/Subscription/Classic/Subscription.cs +++ b/src/Opc.Ua.Client/Subscription/Classic/Subscription.cs @@ -1020,7 +1020,7 @@ public bool PublishingStopped /// public async Task CreateAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(false); // The subscription is not created, so a server id an item still @@ -1082,7 +1082,7 @@ public async Task CreateAsync(CancellationToken ct = default) /// public async Task DeleteAsync(bool silent, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); if (!silent) { VerifySessionAndSubscriptionState(true); @@ -1170,7 +1170,7 @@ internal async Task ResetForSessionRecreateAsync() /// public async Task ModifyAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); // modify the subscription. @@ -1206,7 +1206,7 @@ public async Task ModifyAsync(CancellationToken ct = default) /// public async Task SetPublishingModeAsync(bool enabled, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); // modify the subscription. @@ -1244,7 +1244,7 @@ public async Task RepublishAsync( uint sequenceNumber, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); RepublishResponse response = await Session @@ -1259,7 +1259,7 @@ public async Task RepublishAsync( /// public async Task ApplyChangesAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); await DeleteItemsAsync(ct).ConfigureAwait(false); await ModifyItemsAsync(ct).ConfigureAwait(false); await CreateItemsAsync(ct).ConfigureAwait(false); @@ -1270,7 +1270,7 @@ public async Task ApplyChangesAsync(CancellationToken ct = default) /// public async Task ResolveItemNodeIdsAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); // collect list of browse paths. @@ -1320,7 +1320,7 @@ public async Task> CreateItemsAsync(CancellationToken ct return itemsToCreate; } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); try { // create monitored items. @@ -1394,7 +1394,7 @@ public async Task> ModifyItemsAsync(CancellationToken ct return itemsToModify; } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); // modify the subscription. ModifyMonitoredItemsResponse response = await Session .ModifyMonitoredItemsAsync(null, Id, TimestampsToReturn, requestItems, ct) @@ -1448,7 +1448,7 @@ public async Task> DeleteItemsAsync( } } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); ArrayOf monitoredItemIds = itemsToDelete.ConvertAll( monitoredItem => monitoredItem.Status.Id); @@ -1583,7 +1583,7 @@ await session.SetTriggeringAsync( ArrayOf monitoredItems, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); if (monitoredItems.Count == 0) @@ -1647,7 +1647,7 @@ public async Task SetTriggeringAsync( throw new ArgumentNullException(nameof(triggeringItem)); } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); if (!triggeringItem.Status.Created) @@ -1740,7 +1740,7 @@ public async Task SetTriggeringAsync( /// public async Task ConditionRefreshAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); ArrayOf methodsToCall = @@ -1776,7 +1776,7 @@ public async Task ConditionRefresh2Async( uint monitoredItemId, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); ArrayOf methodsToCall = @@ -1845,7 +1845,7 @@ public async Task TransferAsync( bool deleteOnDefinitiveFailure, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); if (Created) { // handle the case when the client has the subscription template and reconnects @@ -2468,7 +2468,7 @@ public void RemoveItems(IEnumerable monitoredItems) /// public async Task ResendDataAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); VerifySessionAndSubscriptionState(true); try { @@ -2494,7 +2494,7 @@ await Session.CallAsync(ObjectIds.Server, MethodIds.Server_ResendData, ct, Id) ArrayOf )> GetMonitoredItemsAsync(CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); (StatusCode status, ArrayOf serverHandles, ArrayOf clientHandles) = await CallGetMonitoredItemsAsync(ct).ConfigureAwait(false); return (StatusCode.IsGood(status), serverHandles, clientHandles); @@ -2547,7 +2547,7 @@ ArrayOf ClientHandles uint lifetimeInHours, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); uint revisedLifetimeInHours = lifetimeInHours; VerifySession(); @@ -3926,6 +3926,28 @@ private void PublishingStateChanged( private uint m_lastSequenceNumberProcessed; private bool m_resyncLastSequenceNumberProcessed; private LinkedList? m_incomingMessages; + + /// + /// The activity source for service calls, resolved once per telemetry + /// context: the GetActivitySource extension walks the stack to find the + /// calling assembly, which is too expensive to repeat for every call. + /// + private ActivitySource CachedActivitySource + { + get + { + ITelemetryContext? telemetry = m_telemetry; + Tuple? cached = m_activitySource; + if (cached == null || !ReferenceEquals(cached.Item1, telemetry)) + { + cached = Tuple.Create(telemetry, telemetry.GetActivitySource()); + m_activitySource = cached; + } + return cached.Item2; + } + } + + private Tuple? m_activitySource; private ITelemetryContext? m_telemetry; private ILogger m_logger; private ILogger m_eventLogger; diff --git a/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs b/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs index 94c21576f5..9a648d7a4d 100644 --- a/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs +++ b/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs @@ -897,6 +897,53 @@ public static ArraySegment SymmetricEncryptAndSign( uint tokenId, uint lastSequenceNumber, ISymmetricCryptoProvider? provider) + { + Aes? aes = null; + try + { + return SymmetricEncryptAndSign( + data, + securityPolicy, + encryptingKey, + iv, + signingKey, + hmac, + signOnly, + tokenId, + lastSequenceNumber, + provider, + ref aes); + } + finally + { + aes?.Dispose(); + } + } + + /// + /// Encrypts the buffer like the public overload, reusing an AES instance + /// for the CBC policies. + /// + /// + /// aesCache is the AES instance for kept by the + /// caller (one per channel token and direction), or + /// to have one created and stored there. Creating an instance imports the + /// key, which costs more than encrypting a small message. + /// + /// + /// + internal static ArraySegment SymmetricEncryptAndSign( + ArraySegment data, + SecurityPolicyInfo securityPolicy, + byte[] encryptingKey, + byte[] iv, + byte[]? signingKey, + HMAC? hmac, + bool signOnly, + uint tokenId, + uint lastSequenceNumber, + ISymmetricCryptoProvider? provider, + ref Aes? aesCache) { SymmetricEncryptionAlgorithm algorithm = securityPolicy.SymmetricEncryptionAlgorithm; @@ -1020,23 +1067,14 @@ public static ArraySegment SymmetricEncryptAndSign( } else { -#pragma warning disable CA5401 // Symmetric encryption uses non-default initialization vector - using var aes = Aes.Create(); - - aes.Mode = CipherMode.CBC; - aes.Padding = PaddingMode.None; - aes.Key = encryptingKey; - aes.IV = iv; - - using ICryptoTransform encryptor = aes.CreateEncryptor(); -#pragma warning restore CA5401 - - encryptor.TransformBlock( + AesCbcTransform( + true, + encryptingKey, + iv, dataArray, data.Offset, data.Count, - dataArray, - data.Offset); + ref aesCache); } } @@ -1578,6 +1616,55 @@ public static ArraySegment SymmetricDecryptAndVerify( uint lastSequenceNumber, HMAC? hmac, ISymmetricCryptoProvider? provider) + { + Aes? aes = null; + try + { + return SymmetricDecryptAndVerify( + data, + securityPolicy, + encryptingKey, + iv, + signingKey, + signOnly, + tokenId, + lastSequenceNumber, + hmac, + provider, + ref aes); + } + finally + { + aes?.Dispose(); + } + } + + /// + /// Decrypts and verifies the buffer like the public overload, reusing an + /// AES instance for the CBC policies. + /// + /// + /// aesCache is the AES instance for kept by the + /// caller (one per channel token and direction), or + /// to have one created and stored there. + /// + /// + /// + /// + /// The signature HMAC could not be created. + /// + internal static ArraySegment SymmetricDecryptAndVerify( + ArraySegment data, + SecurityPolicyInfo securityPolicy, + byte[] encryptingKey, + byte[] iv, + byte[]? signingKey, + bool signOnly, + uint tokenId, + uint lastSequenceNumber, + HMAC? hmac, + ISymmetricCryptoProvider? provider, + ref Aes? aesCache) { SymmetricEncryptionAlgorithm algorithm = securityPolicy.SymmetricEncryptionAlgorithm; @@ -1647,21 +1734,14 @@ public static ArraySegment SymmetricDecryptAndVerify( } else { - using var aes = Aes.Create(); - - aes.Mode = CipherMode.CBC; - aes.Padding = PaddingMode.None; - aes.Key = encryptingKey; - aes.IV = iv; - - using ICryptoTransform decryptor = aes.CreateDecryptor(); - - decryptor.TransformBlock( + AesCbcTransform( + false, + encryptingKey, + iv, dataArray, data.Offset, data.Count, - dataArray, - data.Offset); + ref aesCache); } } @@ -1760,6 +1840,72 @@ public static ArraySegment SymmetricDecryptAndVerify( return new ArraySegment(dataArray, 0, data.Offset + data.Count); } + /// + /// Encrypts or decrypts whole AES blocks in place with CBC and no padding. + /// + /// + /// The instance is taken out of while it is in + /// use and put back afterwards, so a concurrent caller creates its own + /// rather than sharing it. A channel encrypts and decrypts one message at a + /// time per direction, so in practice one instance per key is reused. + /// + /// + private static void AesCbcTransform( + bool encrypt, + byte[] key, + byte[] iv, + byte[] buffer, + int offset, + int count, + ref Aes? aesCache) + { + Aes aes = Interlocked.Exchange(ref aesCache, null) ?? CreateCbcAes(key); + try + { +#if NET6_0_OR_GREATER + Span data = buffer.AsSpan(offset, count); + int written = encrypt + ? aes.EncryptCbc(data, iv, data, PaddingMode.None) + : aes.DecryptCbc(data, iv, data, PaddingMode.None); + if (written != count) + { + throw new CryptographicException( + "The cipher did not transform the whole buffer."); + } +#else + aes.IV = iv; +#pragma warning disable CA5401 // Symmetric encryption uses non-default initialization vector + using ICryptoTransform transform = encrypt + ? aes.CreateEncryptor() + : aes.CreateDecryptor(); +#pragma warning restore CA5401 + transform.TransformBlock(buffer, offset, count, buffer, offset); +#endif + } + finally + { + // An instance another caller returned meanwhile is surplus. + Interlocked.Exchange(ref aesCache, aes)?.Dispose(); + } + } + + private static Aes CreateCbcAes(byte[] key) + { + var aes = Aes.Create(); + try + { + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + aes.Key = key; + return aes; + } + catch + { + aes.Dispose(); + throw; + } + } + /// /// Rejects a signed buffer too short to hold its signature. /// diff --git a/src/Opc.Ua.Core/Stack/Https/HttpsTransportChannel.cs b/src/Opc.Ua.Core/Stack/Https/HttpsTransportChannel.cs index d729d90f3d..2b9487871c 100644 --- a/src/Opc.Ua.Core/Stack/Https/HttpsTransportChannel.cs +++ b/src/Opc.Ua.Core/Stack/Https/HttpsTransportChannel.cs @@ -260,7 +260,7 @@ public async ValueTask SendRequestAsync( } IServiceMessageContext context = m_quotas?.MessageContext ?? throw BadNotConnected(); HttpClient client = m_client ?? throw BadNotConnected(); - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); using CancellationTokenSource cts = m_timeProvider.CreateCancellationTokenSource( TimeSpan.FromMilliseconds(OperationTimeout)); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, ct); @@ -888,6 +888,15 @@ private static StatusCode MapSocketError(SocketError error) private X509Certificate2? m_pinnedClientCertX509; private bool m_disposeClient; private bool m_disposed; + + /// + /// The activity source for service calls, resolved once: the + /// GetActivitySource extension walks the stack to find the calling + /// assembly, which is too expensive to repeat for every call. + /// + private ActivitySource CachedActivitySource => m_activitySource ??= m_telemetry.GetActivitySource(); + + private ActivitySource? m_activitySource; private readonly ITelemetryContext m_telemetry; private readonly ILogger m_logger; private readonly TimeProvider m_timeProvider; diff --git a/src/Opc.Ua.Core/Stack/Tcp/ChannelAsyncOperation.cs b/src/Opc.Ua.Core/Stack/Tcp/ChannelAsyncOperation.cs index 20de4169f2..4c87a81f00 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/ChannelAsyncOperation.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/ChannelAsyncOperation.cs @@ -31,6 +31,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Sources; using Microsoft.Extensions.Logging; namespace Opc.Ua.Bindings @@ -70,8 +71,8 @@ public ChannelAsyncOperation( if (timeout is > 0 and not int.MaxValue) { m_timer = m_timeProvider.CreateTimer( - new TimerCallback(OnTimeout), - null, + s_onTimeout, + this, TimeSpan.FromMilliseconds(timeout), Timeout.InfiniteTimeSpan); } @@ -113,6 +114,9 @@ protected virtual void Dispose(bool disposing) } m_tcs = null; } + + // like the cancelled task above: the wait is interrupted. + m_waiter?.TrySetResult(false); } } } @@ -122,7 +126,7 @@ protected virtual void Dispose(bool disposing) /// public bool Complete(T response) { - return InternalComplete(true, response); + return InternalComplete(true, Box(response)); } /// @@ -130,7 +134,21 @@ public bool Complete(T response) /// public bool Complete(bool doNotBlock, T response) { - return InternalComplete(doNotBlock, response); + return InternalComplete(doNotBlock, Box(response)); + } + + /// + /// Boxes a response for . The default value of a + /// value type, the usual response of a write, shares one immutable box. + /// + private static object? Box(T response) + { + if (typeof(T).IsValueType && + EqualityComparer.Default.Equals(response, default!)) + { + return s_boxedDefault; + } + return response; } /// @@ -263,6 +281,7 @@ public async Task EndAsync( if (mustWait) { bool badRequestInterrupted = false; + CancellationTokenRegistration cancellation = default; try { Task awaitableTask = m_tcs!.Task; @@ -271,9 +290,15 @@ public async Task EndAsync( awaitableTask = m_tcs.Task .WaitAsync(TimeSpan.FromMilliseconds(timeout), m_timeProvider, ct); } - else if (ct != default) + else if (ct.CanBeCanceled) { - awaitableTask = m_tcs.Task.WaitAsync(ct); + // Cancellation ends the wait like WaitAsync(ct) would, without + // allocating a second task: the source completes with false, + // which reports BadRequestInterrupted below. A response that + // arrives later finds no source to complete. + cancellation = ct.Register( + static state => ((TaskCompletionSource)state!).TrySetResult(false), + m_tcs); } if (!await awaitableTask.ConfigureAwait(false)) { @@ -290,6 +315,7 @@ public async Task EndAsync( } finally { + cancellation.Dispose(); lock (m_lock) { m_tcs = null; @@ -314,6 +340,45 @@ public async Task EndAsync( } } + /// + /// Waits for the operation like without a timeout and + /// throwing on error, but without allocating tasks for the wait. + /// + /// + /// Only one wait at a time is supported. The continuation runs on the thread + /// pool, never on the thread that completes the operation. + /// + /// Ends the wait with BadRequestInterrupted. + /// + /// BadRequestInterrupted when is cancelled or the + /// operation is disposed while waiting, or the error the operation failed + /// with. + /// + internal ValueTask WaitForCompletionAsync(CancellationToken ct) + { + CompletionWaiter waiter; + lock (m_lock) + { + if (m_completed) + { + return m_error == null + ? default + : new ValueTask(Task.FromException(new ServiceResultException(m_error))); + } + waiter = new CompletionWaiter(this); + m_waiter = waiter; + } + + if (ct.CanBeCanceled) + { + // Runs the callback inline if the token is already cancelled. + waiter.Registration = ct.Register( + static state => ((CompletionWaiter)state!).TrySetResult(false), + waiter); + } + return new ValueTask(waiter, waiter.Version); + } + /// /// Stores additional state information associated with the operation. /// @@ -426,6 +491,8 @@ protected virtual bool InternalComplete(bool doNotBlock, object? result) m_event?.Set(); m_tcs?.TrySetResult(true); + + m_waiter?.TrySetResult(true); } AsyncCallback? callback = m_callback; @@ -481,6 +548,9 @@ protected virtual bool InternalComplete(bool doNotBlock, object? result) return true; } + private static readonly TimerCallback s_onTimeout = + static state => ((ChannelAsyncOperation)state!).OnTimeout(null); + private static readonly object? s_boxedDefault = default(T); private readonly Lock m_lock = new(); private readonly AsyncCallback? m_callback; private readonly object? m_asyncState; @@ -494,6 +564,90 @@ protected virtual bool InternalComplete(bool doNotBlock, object? result) private ServiceResult? m_error; private ITimer? m_timer; private Dictionary? m_properties; + private CompletionWaiter? m_waiter; + + /// + /// Completes the wait of : with + /// when the operation completed, with + /// when the wait was interrupted. + /// + private sealed class CompletionWaiter : IValueTaskSource + { + public CompletionWaiter(ChannelAsyncOperation owner) + { + m_owner = owner; + m_core.RunContinuationsAsynchronously = true; + } + + /// + /// The cancellation registration, disposed when the result is read. + /// + public CancellationTokenRegistration Registration { get; set; } + + public short Version => m_core.Version; + + /// + /// Sets the result once; later calls are ignored. + /// + public void TrySetResult(bool completed) + { + if (Interlocked.Exchange(ref m_resultSet, 1) == 0) + { + m_core.SetResult(completed); + } + } + + public void GetResult(short token) + { + bool completed; + try + { + completed = m_core.GetResult(token); + } + finally + { + Registration.Dispose(); + lock (m_owner.m_lock) + { + if (ReferenceEquals(m_owner.m_waiter, this)) + { + m_owner.m_waiter = null; + } + } + } + + if (!completed) + { + throw new ServiceResultException(StatusCodes.BadRequestInterrupted); + } + + lock (m_owner.m_lock) + { + if (m_owner.m_error != null) + { + throw new ServiceResultException(m_owner.m_error); + } + } + } + + public ValueTaskSourceStatus GetStatus(short token) + { + return m_core.GetStatus(token); + } + + public void OnCompleted( + Action continuation, + object? state, + short token, + ValueTaskSourceOnCompletedFlags flags) + { + m_core.OnCompleted(continuation, state, token, flags); + } + + private readonly ChannelAsyncOperation m_owner; + private ManualResetValueTaskSourceCore m_core; + private int m_resultSet; + } } /// diff --git a/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs b/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs index 4515f8d5bd..0d8ab05caa 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs @@ -64,6 +64,9 @@ private void Dispose(bool disposing) ClientHmac?.Dispose(); ClientHmac = null; + Interlocked.Exchange(ref ServerAes, null)?.Dispose(); + Interlocked.Exchange(ref ClientAes, null)?.Dispose(); + m_localNonce?.Dispose(); m_localNonce = null; m_remoteNonce?.Dispose(); @@ -212,6 +215,19 @@ internal bool IsActivationRequired(TimeProvider timeProvider) /// internal HMAC? ClientHmac { get; set; } + /// + /// The AES instance for with the CBC + /// policies, created on first use. A field so that it can be taken and + /// returned atomically. + /// + internal Aes? ServerAes; + + /// + /// The AES instance for with the CBC + /// policies, created on first use. + /// + internal Aes? ClientAes; + /// /// Takes ownership of the actual key-agreement objects while a reconnect is handed off. /// diff --git a/src/Opc.Ua.Core/Stack/Tcp/TcpByteTransport.cs b/src/Opc.Ua.Core/Stack/Tcp/TcpByteTransport.cs index 6ef93566ea..e7eb5d927a 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/TcpByteTransport.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/TcpByteTransport.cs @@ -31,6 +31,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -219,42 +220,21 @@ public async ValueTask ConnectAsync(Uri url, CancellationToken ct) } /// +#if NET6_0_OR_GREATER + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] +#endif public async ValueTask SendChunkAsync(ReadOnlyMemory chunk, CancellationToken ct) { Socket socket = RequireConnectedSocket(); - using var linkedCts = - CancellationTokenSource.CreateLinkedTokenSource(ct, m_sendCancellation.Token); - await m_sendLock.WaitAsync(linkedCts.Token).ConfigureAwait(false); + // The channel sends with CancellationToken.None, so link only when the + // caller's token can be cancelled. + using CancellationTokenSource? linkedCts = ct.CanBeCanceled + ? CancellationTokenSource.CreateLinkedTokenSource(ct, m_sendCancellation.Token) + : null; + await m_sendLock.WaitAsync(linkedCts?.Token ?? m_sendCancellation.Token).ConfigureAwait(false); try { - int sent = 0; - while (sent < chunk.Length) - { - ReadOnlyMemory slice = chunk[sent..]; -#if NET5_0_OR_GREATER - int n = await socket - .SendAsync(slice, SocketFlags.None, ct) - .ConfigureAwait(false); -#else - if (!MemoryMarshal.TryGetArray(slice, out ArraySegment seg)) - { - // Fall back to a copy when the memory does not wrap an array - // (rare in this code path: BufferManager always rents arrays). - byte[] tmp = slice.ToArray(); - seg = new ArraySegment(tmp, 0, tmp.Length); - } - int n = await socket - .SendAsync(seg, SocketFlags.None) - .ConfigureAwait(false); -#endif - if (n <= 0) - { - throw ServiceResultException.Create( - StatusCodes.BadConnectionClosed, - "Remote side closed the connection while sending."); - } - sent += n; - } + await SendMemoryAsync(socket, chunk, ct).ConfigureAwait(false); } finally { @@ -263,6 +243,9 @@ public async ValueTask SendChunkAsync(ReadOnlyMemory chunk, CancellationTo } /// +#if NET6_0_OR_GREATER + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] +#endif public async ValueTask SendChunkAsync(BufferCollection buffers, CancellationToken ct) { if (buffers == null) @@ -270,17 +253,31 @@ public async ValueTask SendChunkAsync(BufferCollection buffers, CancellationToke throw new ArgumentNullException(nameof(buffers)); } Socket socket = RequireConnectedSocket(); - using var linkedCts = - CancellationTokenSource.CreateLinkedTokenSource(ct, m_sendCancellation.Token); - await m_sendLock.WaitAsync(linkedCts.Token).ConfigureAwait(false); + using CancellationTokenSource? linkedCts = ct.CanBeCanceled + ? CancellationTokenSource.CreateLinkedTokenSource(ct, m_sendCancellation.Token) + : null; + await m_sendLock.WaitAsync(linkedCts?.Token ?? m_sendCancellation.Token).ConfigureAwait(false); try { + if (buffers.Count == 1) + { + // A single-chunk message: the memory overload avoids the Task + // and the scatter/gather list of the vectored send. + ArraySegment buffer = buffers[0]; + await SendMemoryAsync( + socket, + new ReadOnlyMemory(buffer.Array, buffer.Offset, buffer.Count), + ct).ConfigureAwait(false); + return; + } + // Socket.SendAsync(IList>) is a vectored send // available on all targets but does not accept a CancellationToken, // so we use it directly and rely on Close()/Dispose() for cancel. await SendAllAsync( buffers, - pending => socket.SendAsync(pending, SocketFlags.None)) + socket, + static (s, pending) => s.SendAsync(pending, SocketFlags.None)) .ConfigureAwait(false); } finally @@ -289,7 +286,48 @@ await SendAllAsync( } } +#if NET6_0_OR_GREATER + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] +#endif + private static async ValueTask SendMemoryAsync( + Socket socket, + ReadOnlyMemory chunk, + CancellationToken ct) + { + int sent = 0; + while (sent < chunk.Length) + { + ReadOnlyMemory slice = chunk[sent..]; +#if NET5_0_OR_GREATER + int n = await socket + .SendAsync(slice, SocketFlags.None, ct) + .ConfigureAwait(false); +#else + if (!MemoryMarshal.TryGetArray(slice, out ArraySegment seg)) + { + // Fall back to a copy when the memory does not wrap an array + // (rare in this code path: BufferManager always rents arrays). + byte[] tmp = slice.ToArray(); + seg = new ArraySegment(tmp, 0, tmp.Length); + } + int n = await socket + .SendAsync(seg, SocketFlags.None) + .ConfigureAwait(false); +#endif + if (n <= 0) + { + throw ServiceResultException.Create( + StatusCodes.BadConnectionClosed, + "Remote side closed the connection while sending."); + } + sent += n; + } + } + /// +#if NET6_0_OR_GREATER + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] +#endif public async ValueTask> ReceiveChunkAsync(CancellationToken ct) { Socket socket = RequireConnectedSocket(); @@ -433,6 +471,20 @@ internal static async ValueTask SendAllAsync( throw new ArgumentNullException(nameof(sendAsync)); } + await SendAllAsync(buffers, sendAsync, static (send, pending) => send(pending)) + .ConfigureAwait(false); + } + + /// + /// Sends all buffers, passing the state to a static send delegate so that + /// the per-send path does not allocate a closure. + /// + /// The state passed to the send delegate. + private static async ValueTask SendAllAsync( + BufferCollection buffers, + TState state, + Func>, Task> sendAsync) + { long totalSize = 0; foreach (ArraySegment buffer in buffers) { @@ -445,7 +497,7 @@ internal static async ValueTask SendAllAsync( int startIndex = 0; while (remaining > 0) { - int sent = await sendAsync(pending).ConfigureAwait(false); + int sent = await sendAsync(state, pending).ConfigureAwait(false); if (sent <= 0) { throw ServiceResultException.Create( @@ -511,6 +563,9 @@ private static int AdvanceBuffers( return startIndex; } +#if NET6_0_OR_GREATER + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] +#endif private static async ValueTask ReadExactAsync( Socket socket, byte[] buffer, diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs index 3ccbe64b88..964dcdef37 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs @@ -28,6 +28,7 @@ * ======================================================================*/ using System; +using System.Buffers.Binary; using System.IO; using System.Security.Cryptography; using System.Text; @@ -634,10 +635,9 @@ protected BufferCollection WriteSymmetricMessage( continue; } -#pragma warning disable CA2000 // Stream is disposed by the BinaryEncoder (leaveOpen: false) - var strm = new MemoryStream(chunkArray, 0, sendBufferSize); - using var encoder = new BinaryEncoder(strm, Quotas.MessageContext, false); -#pragma warning restore CA2000 + // The headers are fixed-size little-endian fields in front of the + // body, written directly rather than through an encoder per chunk. + uint chunkType; // check if the message needs to be aborted. The segment holds // only the body (it starts after the headers), which is what @@ -647,7 +647,7 @@ protected BufferCollection WriteSymmetricMessage( messageSize + chunkToProcess.Count, ii + 1)) { - encoder.WriteUInt32(null, messageType | TcpMessageType.Abort); + chunkType = messageType | TcpMessageType.Abort; // Replace the body in the chunk with an error message. // The encoder is bounded by the room the chunk has for a @@ -679,12 +679,12 @@ protected BufferCollection WriteSymmetricMessage( // check if the message is complete. else if (ii == chunksToProcess.Count - 1) { - encoder.WriteUInt32(null, messageType | TcpMessageType.Final); + chunkType = messageType | TcpMessageType.Final; } // more chunks to follow. else { - encoder.WriteUInt32(null, messageType | TcpMessageType.Intermediate); + chunkType = messageType | TcpMessageType.Intermediate; } int count = 0; @@ -713,16 +713,18 @@ protected BufferCollection WriteSymmetricMessage( count += TcpMessageLimits.SymmetricHeaderSize; - encoder.WriteUInt32(null, (uint)count); - encoder.WriteUInt32(null, ChannelId); - encoder.WriteUInt32(null, token.TokenId); + Span header = chunkArray.AsSpan(0, headerSize); + BinaryPrimitives.WriteUInt32LittleEndian(header, chunkType); + BinaryPrimitives.WriteUInt32LittleEndian(header[4..], (uint)count); + BinaryPrimitives.WriteUInt32LittleEndian(header[8..], ChannelId); + BinaryPrimitives.WriteUInt32LittleEndian(header[12..], token.TokenId); uint sequenceNumber = GetNewSequenceNumber(); - encoder.WriteUInt32(null, sequenceNumber); - encoder.WriteUInt32(null, requestId); + BinaryPrimitives.WriteUInt32LittleEndian(header[16..], sequenceNumber); + BinaryPrimitives.WriteUInt32LittleEndian(header[20..], requestId); - // skip body. - strm.Seek(chunkToProcess.Count, SeekOrigin.Current); + // the headers are followed by the body. + int position = headerSize + chunkToProcess.Count; // update message size count. messageSize += chunkToProcess.Count; @@ -734,7 +736,7 @@ protected BufferCollection WriteSymmetricMessage( dataToSend = new ArraySegment( chunkArray, TcpMessageLimits.SymmetricHeaderSize, - encoder.Position - TcpMessageLimits.SymmetricHeaderSize); + position - TcpMessageLimits.SymmetricHeaderSize); dataToSend = EncryptAndSign(token, dataToSend, isRequest); } @@ -743,7 +745,7 @@ protected BufferCollection WriteSymmetricMessage( dataToSend = new ArraySegment( chunkArray, 0, - encoder.Position); + position); } // add the header into chunk. @@ -779,7 +781,8 @@ private ArraySegment EncryptAndSign( SecurityMode == MessageSecurityMode.Sign, token.TokenId, LastSentSequenceNumber, // already incremented to create this message. need the last one sent. - m_symmetricProvider); + m_symmetricProvider, + ref useClientKeys ? ref token.ClientAes : ref token.ServerAes); } /// @@ -793,11 +796,12 @@ protected ArraySegment ReadSymmetricMessage( out uint requestId, out uint sequenceNumber) { - using var decoder = new BinaryDecoder(buffer, Quotas.MessageContext); - uint messageType = decoder.ReadUInt32(null); - uint messageSize = decoder.ReadUInt32(null); - uint channelId = decoder.ReadUInt32(null); - uint tokenId = decoder.ReadUInt32(null); + // The headers are fixed-size little-endian fields, read directly + // rather than through a decoder per chunk. + _ = ReadHeaderField(buffer, 0); // message type + _ = ReadHeaderField(buffer, 4); // message size + uint channelId = ReadHeaderField(buffer, 8); + uint tokenId = ReadHeaderField(buffer, 12); // ensure the channel is valid. if (channelId != ChannelId) @@ -882,7 +886,7 @@ protected ArraySegment ReadSymmetricMessage( token.CreatedAtTimestamp); } - int headerSize = decoder.Position; + int headerSize = TcpMessageLimits.SymmetricHeaderSize; // ArraySegment.Array is non-null because the buffer comes from BufferManager. byte[] bufferArray = buffer.GetArray(); @@ -911,8 +915,8 @@ protected ArraySegment ReadSymmetricMessage( } // extract request id and sequence number. - sequenceNumber = decoder.ReadUInt32(null); - requestId = decoder.ReadUInt32(null); + sequenceNumber = ReadHeaderField(buffer, headerSize); + requestId = ReadHeaderField(buffer, headerSize + 4); headerSize += TcpMessageLimits.SequenceHeaderSize; @@ -923,6 +927,29 @@ protected ArraySegment ReadSymmetricMessage( dataToProcess.Count - headerSize); } + /// + /// Reads a 32-bit header field of a message chunk. + /// + /// The chunk ends before the + /// field, with the decoding error a decoder reports. + private uint ReadHeaderField(ArraySegment buffer, int offset) + { + if (buffer.Count - offset >= sizeof(uint)) + { + return BinaryPrimitives.ReadUInt32LittleEndian( + buffer.AsSpan(offset, sizeof(uint))); + } + + // Too short: the decoder raises the same error as when it read the + // whole header. + using var decoder = new BinaryDecoder( + buffer.GetArray(), + buffer.Offset + Math.Min(offset, buffer.Count), + Math.Max(buffer.Count - offset, 0), + Quotas.MessageContext); + return decoder.ReadUInt32(null); + } + private ArraySegment DecryptAndVerify( ChannelToken token, ArraySegment dataToDecrypt, @@ -939,7 +966,8 @@ private ArraySegment DecryptAndVerify( token.TokenId, m_remoteSequenceNumber, useClientKeys ? token.ClientHmac : token.ServerHmac, - m_symmetricProvider); + m_symmetricProvider, + ref useClientKeys ? ref token.ClientAes : ref token.ServerAes); } private static readonly byte[] s_hkdfClientLabel = Encoding.UTF8.GetBytes("opcua-client"); diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs index adf4af4461..96038aad90 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.cs @@ -1326,8 +1326,7 @@ protected void BeginWriteMessage(ArraySegment buffer, object? state) // holding it: started inline, the synchronous prologue would run on // the caller's stack, disclaim the caller's own entitlement and then // block on a gate that this very thread holds. - byte[] backingBuffer = buffer.GetArray(); - QueueWrite(() => WriteSingleChunkAsync(transport, chunk, backingBuffer, state)); + QueueWrite(new PendingWrite(transport, chunk, buffer.GetArray(), null, state)); } /// @@ -1358,7 +1357,7 @@ protected void BeginWriteMessage(BufferCollection buffers, object? state) // Queued rather than started inline, for the reason given in // BeginWriteMessage(ArraySegment{byte}, object). - QueueWrite(() => WriteBuffersAsync(transport, buffers, state)); + QueueWrite(new PendingWrite(transport, default, null, buffers, state)); } /// @@ -1372,34 +1371,110 @@ protected void BeginWriteMessage(BufferCollection buffers, object? state) /// required because sequence numbers are assigned under that gate, and a /// peer rejects a chunk whose sequence number does not follow its /// predecessor. Independently queued work items carry no such guarantee, - /// so each write is appended to a chain and cannot start before the one - /// issued before it has finished. + /// so the writes are queued and a single writer on the thread pool sends + /// them one after the other; a write cannot start before the one issued + /// before it has finished. /// - private void QueueWrite(Func write) - { - lock (m_writeChainLock) - { - // No ExecuteSynchronously on the write itself: the continuation - // must reach the pool rather than run on whichever thread - // completed its predecessor, which may be the caller's. The - // trailing continuation observes any fault so that a write which - // fails outside its own error handling cannot surface later as an - // unobserved task exception. - m_writeChain = m_writeChain - .ContinueWith( - static (_, state) => ((Func)state!)(), - write, - CancellationToken.None, - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default) - .Unwrap() - .ContinueWith( - static completed => _ = completed.Exception, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously | - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default); + private void QueueWrite(PendingWrite write) + { + lock (m_writeQueueLock) + { + m_writeQueue.Enqueue(write); + if (m_writerRunning) + { + return; + } + m_writerRunning = true; + } + + // The execution context is not needed by the write and is not flowed. + // The write goes to the local queue of the issuing thread, which picks + // it up as soon as it returns to the pool (typically to await the + // response), like the task continuation this replaces did. +#if NETCOREAPP3_0_OR_GREATER + ThreadPool.UnsafeQueueUserWorkItem(s_runWrites, this, preferLocal: true); +#else + ThreadPool.UnsafeQueueUserWorkItem(s_runWrites, this); +#endif + } + +#if NETCOREAPP3_0_OR_GREATER + private static readonly Action s_runWrites = + static channel => _ = channel.RunWritesAsync(); +#else + private static readonly WaitCallback s_runWrites = + static state => _ = ((UaSCUaBinaryChannel)state!).RunWritesAsync(); +#endif + + /// + /// Sends the queued writes in order until the queue is empty. + /// + private async Task RunWritesAsync() + { + while (true) + { + PendingWrite write; + lock (m_writeQueueLock) + { + if (m_writeQueue.Count == 0) + { + m_writerRunning = false; + return; + } + write = m_writeQueue.Dequeue(); + } + + Task pending = write.Buffers != null + ? WriteBuffersAsync(write.Transport, write.Buffers, write.State) + : WriteSingleChunkAsync(write.Transport, write.Chunk, write.BackingBuffer!, write.State); + if (pending.IsCompleted) + { + // Observe a fault that escaped the write's own error handling, so + // that it cannot surface later as an unobserved task exception. + _ = pending.Exception; + continue; + } + try + { + await pending.ConfigureAwait(false); + } + catch + { + // Observed only, as above; the write reported its own result. + } + + // The await resumed on whichever thread completed the write. The + // next write must start on the pool, not on that thread, which may + // be one that holds a gate (an in-process peer, for instance). + await Task.Yield(); + } + } + + /// + /// A write waiting for its turn: either one contiguous chunk with the + /// buffer to return, or a collection of buffers. + /// + private readonly struct PendingWrite + { + public PendingWrite( + IUaSCByteTransport transport, + ReadOnlyMemory chunk, + byte[]? backingBuffer, + BufferCollection? buffers, + object? state) + { + Transport = transport; + Chunk = chunk; + BackingBuffer = backingBuffer; + Buffers = buffers; + State = state; } + + public IUaSCByteTransport Transport { get; } + public ReadOnlyMemory Chunk { get; } + public byte[]? BackingBuffer { get; } + public BufferCollection? Buffers { get; } + public object? State { get; } } /// @@ -1546,18 +1621,46 @@ private async Task WriteBuffersAsync( /// it. /// /// - /// The write chain exists to keep chunks in the order their sequence + /// The write queue exists to keep chunks in the order their sequence /// numbers were assigned, which only concerns reaching the transport. - /// Reporting completion does not, and some channels implement it by - /// entering the gate — so leaving it in the chain would make every - /// subsequent write on the channel wait for a contended gate, throttling - /// a busy publisher badly enough to change its behaviour. + /// Reporting completion does not, and the client channel implements a + /// failed write by entering the gate — so reporting a failure inline + /// would make every subsequent write on the channel wait for a + /// contended gate, throttling a busy publisher badly enough to change + /// its behaviour. A successful write only updates counters and returns + /// buffers, so it is reported inline. /// private void ReportWriteComplete( BufferCollection? buffers, object? state, int sent, ServiceResult result) + { + if (ServiceResult.IsGood(result)) + { + try + { + HandleWriteComplete(buffers, state, sent, result); + } + catch (Exception ex) + { + m_logger.UaSCChannelWriteCompletionFailed(ex, ChannelId); + } + return; + } + + ReportWriteFailure(buffers, state, sent, result); + } + + /// + /// Reports a failed write on the thread pool. A separate method, so that + /// the closure is only allocated for a failure. + /// + private void ReportWriteFailure( + BufferCollection? buffers, + object? state, + int sent, + ServiceResult result) { _ = Task.Run(() => { @@ -1979,8 +2082,9 @@ public void UpdateLastActiveTime() /// private int m_state; private int m_activeWriteRequests; - private readonly Lock m_writeChainLock = new(); - private Task m_writeChain = Task.CompletedTask; + private readonly Lock m_writeQueueLock = new(); + private readonly Queue m_writeQueue = new(); + private bool m_writerRunning; private long m_lastActiveTimestamp; private readonly string m_contextId; private readonly ILogger m_logger; diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryClientChannel.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryClientChannel.cs index a8c872cc60..8d5302449b 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryClientChannel.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryClientChannel.cs @@ -197,7 +197,7 @@ protected override void Dispose(bool disposing) /// public async ValueTask ConnectAsync(Uri url, int timeout, CancellationToken ct) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); if (url == null) { throw new ArgumentNullException(nameof(url)); @@ -323,7 +323,7 @@ public async ValueTask ConnectAsync(Uri url, int timeout, CancellationToken ct) /// public async Task CloseAsync(int timeout, CancellationToken ct = default) { - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); WriteOperation? operation = InternalClose(timeout); // wait for the close to succeed. @@ -365,7 +365,7 @@ public async ValueTask SendRequestAsync( throw new ArgumentException("Timeout must be greater than zero.", nameof(timeout)); } - using Activity? activity = m_telemetry.StartActivity(); + using Activity? activity = CachedActivitySource.StartActivity(); WriteOperation? operation = null; using (Gate.Enter()) { @@ -398,7 +398,7 @@ public async ValueTask SendRequestAsync( } try { - await operation.EndAsync(int.MaxValue, true, ct).ConfigureAwait(false); + await operation.WaitForCompletionAsync(ct).ConfigureAwait(false); } finally { @@ -951,9 +951,11 @@ protected override void HandleWriteComplete( int bytesWritten, ServiceResult result) { - using (Gate.Enter()) + // Only a failed write needs the gate. Successful writes are reported + // inline by the writer and must not wait for it. + if (state is WriteOperation operation && ServiceResult.IsBad(result)) { - if (state is WriteOperation operation && ServiceResult.IsBad(result)) + using (Gate.Enter()) { operation.Fault(result); ForceReconnectCore(result); @@ -1992,6 +1994,15 @@ private bool ProcessResponseMessage(uint messageType, ArraySegment message private readonly AsyncCallback m_handshakeComplete; private List? m_queuedOperations; private readonly ILogger m_logger; + + /// + /// The activity source for service calls, resolved once: the + /// GetActivitySource extension walks the stack to find the calling + /// assembly, which is too expensive to repeat for every call. + /// + private ActivitySource CachedActivitySource => m_activitySource ??= m_telemetry.GetActivitySource(); + + private ActivitySource? m_activitySource; private readonly ITelemetryContext m_telemetry; private byte[]? m_oscRequestSignature; private readonly MessageSecurityMode m_requestedSecurityMode; diff --git a/src/Opc.Ua.Types/Encoders/BinaryEncoder.cs b/src/Opc.Ua.Types/Encoders/BinaryEncoder.cs index d81baca9f9..8c0963ba98 100644 --- a/src/Opc.Ua.Types/Encoders/BinaryEncoder.cs +++ b/src/Opc.Ua.Types/Encoders/BinaryEncoder.cs @@ -2526,7 +2526,7 @@ private static byte GetNodeIdEncoding(IdType idType, in NodeId nodeId, int names encoding = NodeIdEncodingBits.ByteString; break; default: - uint id = Convert.ToUInt32(nodeId.NumericIdentifier, CultureInfo.InvariantCulture); + uint id = nodeId.NumericIdentifier; if (id <= byte.MaxValue && namespaceIndex == 0) { @@ -2544,7 +2544,7 @@ private static byte GetNodeIdEncoding(IdType idType, in NodeId nodeId, int names break; } - return Convert.ToByte(encoding, CultureInfo.InvariantCulture); + return (byte)encoding; } /// diff --git a/tests/Opc.Ua.Client.Tests/Session/SessionKeepAliveTests.cs b/tests/Opc.Ua.Client.Tests/Session/SessionKeepAliveTests.cs index 22a2ca1606..74ee0def92 100644 --- a/tests/Opc.Ua.Client.Tests/Session/SessionKeepAliveTests.cs +++ b/tests/Opc.Ua.Client.Tests/Session/SessionKeepAliveTests.cs @@ -28,6 +28,8 @@ * ======================================================================*/ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Time.Testing; using Moq; using NUnit.Framework; @@ -287,13 +289,78 @@ public void KeepAliveReadAfterTheChannelIsDetachedFailsWithBadSecureChannelClose Assert.That(sre.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed)); } - private static KeepAliveTestSession CreateSession(TimeProvider timeProvider) + /// + /// Responses do not move the keep alive timer each time; the timer moves + /// itself when it fires. The keep alive read must still be sent exactly + /// one interval after the last response, and not while responses keep + /// arriving. + /// + [Test] + [CancelAfter(30000)] + public async Task TheKeepAliveReadIsSentOneIntervalAfterTheLastResponseAsync() + { + var timeProvider = new FakeTimeProvider(); + using var readSent = new SemaphoreSlim(0); + using KeepAliveTestSession session = CreateSession( + timeProvider, + channel => channel + .Setup(c => c.SendRequestAsync(It.IsAny(), It.IsAny())) + .Returns((IServiceRequest request, CancellationToken _) => + { + readSent.Release(); + return new ValueTask(CreateKeepAliveResponse(request)); + })); + session.SessionCreated(new NodeId(1u, 1), new NodeId(2u, 1)); + + // the first keep alive read is sent right away. + await session.StartKeepAliveTimerAsync().ConfigureAwait(false); + Assert.That(await readSent.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false), Is.True); + + // a response every 2 s for 20 s: no keep alive read. + for (int ii = 0; ii < 10; ii++) + { + timeProvider.Advance(TimeSpan.FromSeconds(2)); + session.CompleteRequest(StatusCodes.Good); + Assert.That( + await readSent.WaitAsync(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false), + Is.False, + $"the last response was received {ii * 2000} ms after the previous one"); + } + + // traffic stops: nothing until a full interval after the last response. + timeProvider.Advance(TimeSpan.FromMilliseconds(kKeepAliveInterval - 10)); + Assert.That( + await readSent.WaitAsync(TimeSpan.FromMilliseconds(200)).ConfigureAwait(false), + Is.False); + + timeProvider.Advance(TimeSpan.FromMilliseconds(20)); + Assert.That(await readSent.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false), Is.True); + } + + private static ReadResponse CreateKeepAliveResponse(IServiceRequest request) + { + return new ReadResponse + { + ResponseHeader = new ResponseHeader + { + RequestHandle = request.RequestHeader.RequestHandle, + ServiceResult = StatusCodes.Good, + Timestamp = DateTime.UtcNow + }, + Results = new[] { new DataValue(new Variant((int)ServerState.Running)) }.ToArrayOf() + }; + } + + private static KeepAliveTestSession CreateSession( + TimeProvider timeProvider, + Action>? setupChannel = null) { ITelemetryContext telemetry = NUnitTelemetryContext.Create(); ApplicationConfiguration configuration = CreateClientConfiguration(telemetry); IServiceMessageContext messageContext = configuration.CreateMessageContext(); var channel = new Mock(); channel.SetupGet(c => c.MessageContext).Returns(messageContext); + setupChannel?.Invoke(channel); var session = new KeepAliveTestSession( channel.Object, diff --git a/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs b/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs new file mode 100644 index 0000000000..5e47a40c42 --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs @@ -0,0 +1,231 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Security.Cryptography; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Bindings; + +namespace Opc.Ua.Core.Tests.Security.Certificates +{ + /// + /// The AES instance a channel token keeps per key for the CBC policies. + /// + [TestFixture] + [Category("CryptoUtils")] + [Parallelizable] + [SetCulture("en-us")] + public class CryptoUtilsAesCacheTests + { + private const int kHeaderSize = 24; + private const int kBlockSize = 16; + + /// + /// Encrypting through a cached instance gives the same cipher text as a + /// fresh one, also when the instance is reused with another IV. + /// + [Test] + public void CachedInstanceEncryptsLikeAFreshOne() + { + byte[] key = CreateBytes(32, 1); + Aes? cache = null; + try + { + for (int ii = 0; ii < 3; ii++) + { + byte[] iv = CreateBytes(kBlockSize, (byte)(10 + ii)); + byte[] body = CreateBytes(37 + ii, (byte)(50 + ii)); + + byte[] expected = EncryptWithoutCache(body, key, iv); + byte[] actual = Encrypt(body, key, iv, ref cache); + + Assert.That(actual, Is.EqualTo(expected), $"message {ii}"); + Assert.That(cache, Is.Not.Null); + } + } + finally + { + cache?.Dispose(); + } + } + + /// + /// Messages encrypted and decrypted through cached instances round-trip. + /// + [Test] + public void CachedInstancesRoundTripManyMessages() + { + byte[] key = CreateBytes(32, 3); + byte[] iv = CreateBytes(kBlockSize, 4); + Aes? encryptCache = null; + Aes? decryptCache = null; + try + { + for (int bodySize = 0; bodySize < 40; bodySize++) + { + byte[] body = CreateBytes(bodySize, (byte)bodySize); + byte[] buffer = new byte[512]; + Array.Copy(body, 0, buffer, kHeaderSize, bodySize); + + ArraySegment encrypted = CryptoUtils.SymmetricEncryptAndSign( + new ArraySegment(buffer, kHeaderSize, bodySize), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + null, + false, + 0, + 0, + null, + ref encryptCache); + + ArraySegment decrypted = CryptoUtils.SymmetricDecryptAndVerify( + new ArraySegment(buffer, kHeaderSize, encrypted.Count - kHeaderSize), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + false, + 0, + 0, + null, + null, + ref decryptCache); + + Assert.That(decrypted, Has.Count.EqualTo(kHeaderSize + bodySize)); + byte[] roundTripped = new byte[bodySize]; + Array.Copy(buffer, kHeaderSize, roundTripped, 0, bodySize); + Assert.That(roundTripped, Is.EqualTo(body)); + } + } + finally + { + encryptCache?.Dispose(); + decryptCache?.Dispose(); + } + } + + /// + /// Concurrent users of one cache each take their own instance while it is + /// in use, so every result is correct. + /// + [Test] + public void ConcurrentUsersOfOneCacheGetCorrectResults() + { + byte[] key = CreateBytes(32, 5); + byte[] iv = CreateBytes(kBlockSize, 6); + byte[] body = CreateBytes(300, 7); + byte[] expected = EncryptWithoutCache(body, key, iv); + var holder = new CacheHolder(); + try + { + Parallel.For(0, 256, _ => + { + byte[] actual = Encrypt(body, key, iv, ref holder.Cache); + Assert.That(actual, Is.EqualTo(expected)); + }); + Assert.That(holder.Cache, Is.Not.Null); + } + finally + { + holder.Cache?.Dispose(); + } + } + + /// + /// Disposing a channel token disposes and clears its cached instances. + /// + [Test] + public void DisposingTheTokenReleasesTheCachedInstances() + { + var token = new ChannelToken + { + ClientAes = Aes.Create(), + ServerAes = Aes.Create() + }; + + token.Dispose(); + + Assert.That(token.ClientAes, Is.Null); + Assert.That(token.ServerAes, Is.Null); + } + + private static byte[] Encrypt(byte[] body, byte[] key, byte[] iv, ref Aes? cache) + { + byte[] buffer = new byte[kHeaderSize + body.Length + (2 * kBlockSize)]; + Array.Copy(body, 0, buffer, kHeaderSize, body.Length); + ArraySegment encrypted = CryptoUtils.SymmetricEncryptAndSign( + new ArraySegment(buffer, kHeaderSize, body.Length), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + null, + false, + 0, + 0, + null, + ref cache); + byte[] result = new byte[encrypted.Count]; + Array.Copy(buffer, 0, result, 0, encrypted.Count); + return result; + } + + private static byte[] EncryptWithoutCache(byte[] body, byte[] key, byte[] iv) + { + byte[] buffer = new byte[kHeaderSize + body.Length + (2 * kBlockSize)]; + Array.Copy(body, 0, buffer, kHeaderSize, body.Length); + ArraySegment encrypted = CryptoUtils.SymmetricEncryptAndSign( + new ArraySegment(buffer, kHeaderSize, body.Length), + SecurityPolicyInfo.Basic256Sha256, + key, + iv); + byte[] result = new byte[encrypted.Count]; + Array.Copy(buffer, 0, result, 0, encrypted.Count); + return result; + } + + private static byte[] CreateBytes(int length, byte seed) + { + byte[] bytes = new byte[length]; + for (int ii = 0; ii < length; ii++) + { + bytes[ii] = unchecked((byte)((ii * 31) + seed)); + } + return bytes; + } + + private sealed class CacheHolder + { + public Aes? Cache; + } + } +} diff --git a/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs new file mode 100644 index 0000000000..6ca7c1bb6e --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs @@ -0,0 +1,254 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using Opc.Ua.Bindings; +using Opc.Ua.Tests; + +namespace Opc.Ua.Core.Tests.Stack.Transport +{ + /// + /// The task-free wait the client channel uses for a request's response. + /// + [TestFixture] + [Category("TcpTransport")] + [Parallelizable] + public sealed class ChannelAsyncOperationWaitTests + { + private readonly ILogger m_logger = NUnitTelemetryContext.Create().CreateLogger(); + + [Test] + [CancelAfter(10000)] + public async Task WaitCompletesWhenTheOperationCompletesLaterAsync() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + + ValueTask wait = operation.WaitForCompletionAsync(CancellationToken.None); + Assert.That(wait.IsCompleted, Is.False); + + Assert.That(operation.Complete(true, 7), Is.True); + await wait.ConfigureAwait(false); + + Assert.That(operation.End(0), Is.EqualTo(7)); + } + + [Test] + public void WaitOnACompletedOperationCompletesSynchronously() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + operation.Complete(true, 0); + + ValueTask wait = operation.WaitForCompletionAsync(CancellationToken.None); + + Assert.That(wait.IsCompletedSuccessfully, Is.True); + } + + [Test] + public void WaitOnAFaultedOperationThrowsTheError() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + operation.Fault(new ServiceResult(StatusCodes.BadSecureChannelClosed)); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await operation.WaitForCompletionAsync(CancellationToken.None).ConfigureAwait(false))!; + + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadSecureChannelClosed)); + } + + [Test] + [CancelAfter(10000)] + public void WaitThrowsTheErrorOfAFaultWhileWaiting() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + ValueTask wait = operation.WaitForCompletionAsync(CancellationToken.None); + + operation.Fault(true, new ServiceResult(StatusCodes.BadUnknownResponse)); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await wait.ConfigureAwait(false))!; + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadUnknownResponse)); + } + + [Test] + [CancelAfter(10000)] + public void CancellingTheTokenInterruptsTheWait() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + using var cts = new CancellationTokenSource(); + ValueTask wait = operation.WaitForCompletionAsync(cts.Token); + + cts.Cancel(); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await wait.ConfigureAwait(false))!; + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadRequestInterrupted)); + + // a response that arrives afterwards completes the operation quietly. + Assert.That(operation.Complete(true, 0), Is.True); + } + + [Test] + public void AnAlreadyCancelledTokenInterruptsTheWait() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await operation.WaitForCompletionAsync(cts.Token).ConfigureAwait(false))!; + + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadRequestInterrupted)); + } + + [Test] + [CancelAfter(10000)] + public void DisposingTheOperationInterruptsTheWait() + { + var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + ValueTask wait = operation.WaitForCompletionAsync(CancellationToken.None); + + operation.Dispose(); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await wait.ConfigureAwait(false))!; + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadRequestInterrupted)); + } + + /// + /// The continuation must not run on the thread that completes the + /// operation: that is the channel's receive loop. + /// + [Test] + [CancelAfter(10000)] + public async Task TheContinuationDoesNotRunOnTheCompletingThreadAsync() + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + int completingThread = -1; + int continuationThread = -1; + using var completed = new ManualResetEventSlim(); + + Task waiter = Task.Run(async () => + { + await operation.WaitForCompletionAsync(CancellationToken.None).ConfigureAwait(false); + continuationThread = Environment.CurrentManagedThreadId; + }); + + // let the waiter register before completing on a dedicated thread. + await Task.Delay(50).ConfigureAwait(false); + var thread = new Thread(() => + { + completingThread = Environment.CurrentManagedThreadId; + operation.Complete(true, 0); + // keep the thread busy so an inline continuation would be visible. + completed.Wait(TimeSpan.FromSeconds(5)); + }); + thread.Start(); + + await waiter.ConfigureAwait(false); + completed.Set(); + thread.Join(); + + Assert.That(continuationThread, Is.Not.EqualTo(completingThread)); + } + + /// + /// Completion, cancellation and disposal racing each other always end the + /// wait exactly once, with a success or BadRequestInterrupted. + /// + [Test] + [CancelAfter(60000)] + public async Task CompletionRacingCancellationEndsTheWaitOnceAsync() + { + for (int ii = 0; ii < 2000; ii++) + { + using var operation = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + using var cts = new CancellationTokenSource(); + ValueTask wait = operation.WaitForCompletionAsync(cts.Token); + + using var start = new Barrier(2); + Task complete = Task.Run(() => + { + start.SignalAndWait(); + operation.Complete(true, 0); + }); + Task cancel = Task.Run(() => + { + start.SignalAndWait(); + cts.Cancel(); + }); + + try + { + await wait.ConfigureAwait(false); + } + catch (ServiceResultException ex) + { + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadRequestInterrupted)); + } + await Task.WhenAll(complete, cancel).ConfigureAwait(false); + } + } + + /// + /// The timeout timer uses a shared callback; it still faults the operation. + /// + [Test] + [CancelAfter(10000)] + public void TheTimeoutStillFaultsTheOperation() + { + using var operation = new ChannelAsyncOperation(50, null, null, m_logger); + + ServiceResultException ex = Assert.ThrowsAsync( + async () => await operation.WaitForCompletionAsync(CancellationToken.None).ConfigureAwait(false))!; + + Assert.That(ex.StatusCode, Is.EqualTo((uint)StatusCodes.BadRequestTimeout)); + } + + /// + /// Completing with the default value shares a box; the value still reads + /// back, for value and reference types. + /// + [Test] + public void CompletingWithTheDefaultValueReturnsIt() + { + using var ints = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + ints.Complete(true, 0); + Assert.That(ints.End(0), Is.Zero); + + using var strings = new ChannelAsyncOperation(int.MaxValue, null, null, m_logger); + strings.Complete(true, null); + Assert.That(strings.End(0), Is.Null); + Assert.That(strings.Error, Is.EqualTo(ServiceResult.Good)); + } + } +} diff --git a/tests/Opc.Ua.Core.Tests/Stack/Transport/TcpByteTransportTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Transport/TcpByteTransportTests.cs index 0fbb58d561..f01e8e5c11 100644 --- a/tests/Opc.Ua.Core.Tests/Stack/Transport/TcpByteTransportTests.cs +++ b/tests/Opc.Ua.Core.Tests/Stack/Transport/TcpByteTransportTests.cs @@ -220,6 +220,104 @@ public async Task SendChunkAsyncRoundTripsBytesOverTheSocket() } } + /// + /// A single-chunk message goes through the memory send path; it must send + /// exactly the segment, honouring its offset. + /// + [Test] + public async Task SendChunkAsyncWithOneBufferSendsTheSegmentAsync() + { + (TcpByteTransport client, Socket serverSocket, TcpListener listener) = + await CreateConnectedPairAsync().ConfigureAwait(false); + using var _l = new ListenerScope(listener); + using Socket _s = serverSocket; + using (client) + { + byte[] chunk = BuildValidChunk(TcpMessageType.Hello, 64); + byte[] backing = new byte[chunk.Length + 20]; + Buffer.BlockCopy(chunk, 0, backing, 7, chunk.Length); + var buffers = new BufferCollection { new ArraySegment(backing, 7, chunk.Length) }; + + await client.SendChunkAsync(buffers, CancellationToken.None).ConfigureAwait(false); + + byte[] received = await ReceiveExactlyAsync(serverSocket, chunk.Length).ConfigureAwait(false); + Assert.That(received, Is.EqualTo(chunk)); + } + } + + /// + /// A message of several chunks goes through the vectored send and arrives + /// in order. + /// + [Test] + public async Task SendChunkAsyncWithSeveralBuffersSendsThemInOrderAsync() + { + (TcpByteTransport client, Socket serverSocket, TcpListener listener) = + await CreateConnectedPairAsync().ConfigureAwait(false); + using var _l = new ListenerScope(listener); + using Socket _s = serverSocket; + using (client) + { + byte[] first = BuildValidChunk(TcpMessageType.Hello, 40); + byte[] second = BuildValidChunk(TcpMessageType.Acknowledge, 56); + byte[] third = BuildValidChunk(TcpMessageType.Error, 24); + var buffers = new BufferCollection + { + new ArraySegment(first), + new ArraySegment(second), + new ArraySegment(third) + }; + + await client.SendChunkAsync(buffers, CancellationToken.None).ConfigureAwait(false); + + byte[] received = await ReceiveExactlyAsync( + serverSocket, + first.Length + second.Length + third.Length).ConfigureAwait(false); + byte[] expected = new byte[received.Length]; + Buffer.BlockCopy(first, 0, expected, 0, first.Length); + Buffer.BlockCopy(second, 0, expected, first.Length, second.Length); + Buffer.BlockCopy(third, 0, expected, first.Length + second.Length, third.Length); + Assert.That(received, Is.EqualTo(expected)); + } + } + + /// + /// A cancellable caller token still ends a send that waits for its turn. + /// + [Test] + public async Task SendChunkAsyncWaitingForTheSendLockHonoursTheCallerTokenAsync() + { + (TcpByteTransport client, Socket serverSocket, TcpListener listener) = + await CreateConnectedPairAsync().ConfigureAwait(false); + using var _l = new ListenerScope(listener); + using Socket _s = serverSocket; + using (client) + { + var sendLock = (SemaphoreSlim)typeof(TcpByteTransport) + .GetField("m_sendLock", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(client)!; + sendLock.Wait(); + try + { + using var cts = new CancellationTokenSource(); + var buffers = new BufferCollection + { + new ArraySegment(BuildValidChunk(TcpMessageType.Hello, 32)) + }; + Task send = client.SendChunkAsync(buffers, cts.Token).AsTask(); + Assert.That(send.IsCompleted, Is.False); + + cts.Cancel(); + + Assert.CatchAsync(async () => await send.ConfigureAwait(false)); + } + finally + { + sendLock.Release(); + } + } + } + [Test] public async Task ClosingTransportCompletesAQueuedSendWaiterAsync() { @@ -622,6 +720,21 @@ await transport return (transport, serverSocket, listener); } + private static async Task ReceiveExactlyAsync(Socket socket, int count) + { + byte[] received = new byte[count]; + int total = 0; + while (total < count) + { + int n = await socket + .ReceiveAsync(new ArraySegment(received, total, count - total), SocketFlags.None) + .ConfigureAwait(false); + Assert.That(n, Is.GreaterThan(0)); + total += n; + } + return received; + } + private static ArraySegment[] CopySegments(IList> buffers) { var copy = new ArraySegment[buffers.Count]; diff --git a/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs new file mode 100644 index 0000000000..ece1a3a55d --- /dev/null +++ b/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs @@ -0,0 +1,349 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Bindings; +using Opc.Ua.Tests; + +namespace Opc.Ua.Core.Tests.Stack.Transport +{ + /// + /// The queue that sends a channel's chunks one after the other, in the order + /// they were issued, off the caller's stack. + /// + [TestFixture] + [Category("TcpTransport")] + [Parallelizable] + public sealed class UaSCBinaryChannelWriteQueueTests + { + private readonly ITelemetryContext m_telemetry = NUnitTelemetryContext.Create(); + + /// + /// Writes queued while the first one is still sending are sent in order, + /// never two at a time, and each is reported once. + /// + [Test] + [CancelAfter(30000)] + public async Task QueuedWritesAreSentOneAtATimeInOrderAsync() + { + var transport = new RecordingTransport(); + using var channel = new WriteProbeChannel(m_telemetry, transport); + const int count = 200; + + channel.Write(0, useCollection: true); + await transport.FirstSendStarted.Task.ConfigureAwait(false); + for (int ii = 1; ii < count; ii++) + { + channel.Write(ii, useCollection: ii % 2 == 0); + } + transport.ReleaseFirstSend.SetResult(true); + + await channel.WaitForCompletionsAsync(count).ConfigureAwait(false); + + Assert.That(transport.Sent, Is.EqualTo(Range(count))); + Assert.That(transport.MaxConcurrentSends, Is.EqualTo(1)); + Assert.That(channel.Completed, Is.EquivalentTo(Range(count))); + Assert.That(channel.HasPendingWrites, Is.False); + } + + /// + /// Issuing a write returns before the send starts: the caller may hold the + /// channel gate that completion reporting enters. + /// + [Test] + [CancelAfter(30000)] + public async Task IssuingAWriteDoesNotSendOnTheCallersStackAsync() + { + var transport = new RecordingTransport(); + using var channel = new WriteProbeChannel(m_telemetry, transport); + + int callerThread = Environment.CurrentManagedThreadId; + channel.Write(0, useCollection: false); + Assert.That(transport.Sent, Is.Empty, "the first send blocks until released"); + + await transport.FirstSendStarted.Task.ConfigureAwait(false); + Assert.That(transport.FirstSendThread, Is.Not.EqualTo(callerThread)); + transport.ReleaseFirstSend.SetResult(true); + await channel.WaitForCompletionsAsync(1).ConfigureAwait(false); + } + + /// + /// Writes issued from many threads are all sent and reported, one at a + /// time, and the queue drains completely. + /// + [Test] + [CancelAfter(30000)] + public async Task ConcurrentWritersAreAllSentAsync() + { + var transport = new RecordingTransport { BlockFirstSend = false }; + using var channel = new WriteProbeChannel(m_telemetry, transport); + const int writers = 8; + const int perWriter = 250; + + await Task.WhenAll(Enumerate(writers, w => Task.Run(() => + { + for (int ii = 0; ii < perWriter; ii++) + { + channel.Write((w * perWriter) + ii, useCollection: ii % 3 == 0); + } + }))).ConfigureAwait(false); + + await channel.WaitForCompletionsAsync(writers * perWriter).ConfigureAwait(false); + + Assert.That(transport.Sent, Is.EquivalentTo(Range(writers * perWriter))); + Assert.That(transport.MaxConcurrentSends, Is.EqualTo(1)); + Assert.That(channel.HasPendingWrites, Is.False); + } + + /// + /// A failed send is reported with its error and does not stop the writes + /// queued behind it. + /// + [Test] + [CancelAfter(30000)] + public async Task AFailedSendIsReportedAndLaterWritesStillGoOutAsync() + { + var transport = new RecordingTransport { BlockFirstSend = false, FailIndex = 3 }; + using var channel = new WriteProbeChannel(m_telemetry, transport); + + for (int ii = 0; ii < 6; ii++) + { + channel.Write(ii, useCollection: true); + } + await channel.WaitForCompletionsAsync(6).ConfigureAwait(false); + + Assert.That(transport.Sent, Is.EqualTo(s_sentAroundTheFailure)); + Assert.That(channel.Failed, Is.EqualTo(s_failed)); + Assert.That(channel.HasPendingWrites, Is.False); + } + + private static readonly int[] s_sentAroundTheFailure = [0, 1, 2, 4, 5]; + private static readonly int[] s_failed = [3]; + + private static int[] Range(int count) + { + int[] values = new int[count]; + for (int ii = 0; ii < count; ii++) + { + values[ii] = ii; + } + return values; + } + + private static List Enumerate(int count, Func start) + { + var tasks = new List(count); + for (int ii = 0; ii < count; ii++) + { + tasks.Add(start(ii)); + } + return tasks; + } + + /// + /// A channel that issues writes of a 4-byte index and records how they + /// are reported. + /// + private sealed class WriteProbeChannel : UaSCUaBinaryChannel + { + public WriteProbeChannel(ITelemetryContext telemetry, IUaSCByteTransport transport) + : base( + "writes", + new BufferManager("writes", TcpMessageLimits.DefaultMaxBufferSize, telemetry), + new ChannelQuotas(ServiceMessageContext.Create(telemetry)), + serverCertificate: null, + endpoints: new List(), + securityMode: MessageSecurityMode.None, + securityPolicyUri: SecurityPolicies.None, + telemetry: telemetry) + { + Transport = transport; + } + + public ConcurrentQueue Completed { get; } = new(); + + public ConcurrentQueue Failed { get; } = new(); + + public void Write(int index, bool useCollection) + { + byte[] buffer = BufferManager.TakeBuffer(sizeof(int), "WriteProbeChannel"); + BitConverter.GetBytes(index).CopyTo(buffer, 0); + if (useCollection) + { + BeginWriteMessage( + new BufferCollection { new ArraySegment(buffer, 0, sizeof(int)) }, + index); + } + else + { + BeginWriteMessage(new ArraySegment(buffer, 0, sizeof(int)), index); + } + } + + public async Task WaitForCompletionsAsync(int count) + { + while (Completed.Count + Failed.Count < count) + { + await Task.Delay(5).ConfigureAwait(false); + } + } + + protected override void HandleWriteComplete( + BufferCollection? buffers, + object? state, + int bytesWritten, + ServiceResult result) + { + base.HandleWriteComplete(buffers, state, bytesWritten, result); + (ServiceResult.IsGood(result) ? Completed : Failed).Enqueue((int)state!); + } + } + + /// + /// Records the index carried by each sent chunk and how many sends overlap. + /// The first send can be held until released. + /// + private sealed class RecordingTransport : IUaSCByteTransport + { + public bool BlockFirstSend { get; set; } = true; + + public int FailIndex { get; set; } = -1; + + public TaskCompletionSource FirstSendStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseFirstSend { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int FirstSendThread { get; private set; } + + public List Sent + { + get + { + lock (m_sent) + { + return new List(m_sent); + } + } + } + + public int MaxConcurrentSends => Volatile.Read(ref m_maxConcurrent); + + public EndPoint? LocalEndpoint => null; + + public EndPoint? RemoteEndpoint => null; + + public TransportChannelFeatures Features => TransportChannelFeatures.None; + + public string Implementation => "UA-TEST"; + + public ValueTask ConnectAsync(Uri url, CancellationToken ct) + { + return default; + } + + public ValueTask SendChunkAsync(ReadOnlyMemory chunk, CancellationToken ct) + { + return SendAsync(BitConverter.ToInt32(chunk.ToArray(), 0)); + } + + public ValueTask SendChunkAsync(BufferCollection buffers, CancellationToken ct) + { + ArraySegment segment = buffers[0]; + return SendAsync(BitConverter.ToInt32(segment.Array!, segment.Offset)); + } + + public ValueTask> ReceiveChunkAsync(CancellationToken ct) + { + return new ValueTask>(Task.Delay(Timeout.Infinite, ct) + .ContinueWith(_ => default(ArraySegment), TaskScheduler.Default)); + } + + public void Close() + { + } + + private async ValueTask SendAsync(int index) + { + int concurrent = Interlocked.Increment(ref m_concurrent); + int max; + while (concurrent > (max = Volatile.Read(ref m_maxConcurrent)) && + Interlocked.CompareExchange(ref m_maxConcurrent, concurrent, max) != max) + { + } + try + { + if (Interlocked.Exchange(ref m_started, 1) == 0) + { + FirstSendThread = Environment.CurrentManagedThreadId; + FirstSendStarted.SetResult(true); + if (BlockFirstSend) + { + await ReleaseFirstSend.Task.ConfigureAwait(false); + } + } + else + { + // let a send complete asynchronously now and then. + if (index % 7 == 0) + { + await Task.Yield(); + } + } + + if (index == FailIndex) + { + throw ServiceResultException.Create(StatusCodes.BadConnectionClosed, "send failed"); + } + + lock (m_sent) + { + m_sent.Add(index); + } + } + finally + { + Interlocked.Decrement(ref m_concurrent); + } + } + + private readonly List m_sent = []; + private int m_concurrent; + private int m_maxConcurrent; + private int m_started; + } + } +} From 4928b8066e2b0a5d594407ce3814973eb4984160 Mon Sep 17 00:00:00 2001 From: Marc Date: Sat, 10 Oct 2026 14:48:00 +0200 Subject: [PATCH 19/19] Fix review feedback on client performance changes Keep AES cache shutdown state separate from borrowed cipher ownership so returns racing token disposal release the keyed instances. Cover both encryption directions and concurrent return/disposal. Bound managed subscription setup to one minute, fail on item and subscription errors, and preserve cancellation. Add fake-clock readiness regressions. Make continuation registration deterministic, use a dedicated test lock, seal the AES fixture, and correct process-counter documentation. Validation: 112 targeted core tests and 11 readiness tests pass on each of net10.0 and net48. The harness builds without warnings and encrypted managed-subscription/classic-read smoke scenarios complete. --- perf/README.md | 4 + perf/ServerLoadHarness/Program.cs | 25 +- perf/ServerLoadHarness/SubscriptionSetup.cs | 116 +++++++ .../Security/Certificates/AesCache.cs | 82 +++++ .../Security/Certificates/CryptoUtils.cs | 98 +++--- src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs | 15 +- .../Stack/Tcp/UaSCBinaryChannel.Symmetric.cs | 4 +- .../Opc.Ua.Client.Tests.csproj | 1 + .../Perf/SubscriptionSetupTests.cs | 200 +++++++++++ .../Certificates/CryptoUtilsAesCacheTests.cs | 318 +++++++++++++----- .../ChannelAsyncOperationWaitTests.cs | 21 +- .../UaSCBinaryChannelWriteQueueTests.cs | 5 +- 12 files changed, 719 insertions(+), 170 deletions(-) create mode 100644 perf/ServerLoadHarness/SubscriptionSetup.cs create mode 100644 src/Opc.Ua.Core/Security/Certificates/AesCache.cs create mode 100644 tests/Opc.Ua.Client.Tests/Perf/SubscriptionSetupTests.cs diff --git a/perf/README.md b/perf/README.md index c86430d1a0..b77fc87615 100644 --- a/perf/README.md +++ b/perf/README.md @@ -51,6 +51,10 @@ Environment variables prefixed `SERVER_` are passed to the server process withou for example `SERVER_DOTNET_gcServer=1` to compare garbage collector modes. `DOTNET_gcServer=1` set for the harness itself switches the client. +Managed subscription setup has a one-minute deadline per subscription. Rejected monitored +items and subscription error or deletion states fail the scenario instead of waiting +indefinitely or starting a measurement with incomplete subscriptions. + The result block reports client throughput and latency percentiles, the deltas of both processes for the window (`server:`, `client-process:`), and the allocated bytes and CPU time per request (or per notification) of the server (`per-op:`) and of the client diff --git a/perf/ServerLoadHarness/Program.cs b/perf/ServerLoadHarness/Program.cs index da77f002b7..d449fa97f3 100644 --- a/perf/ServerLoadHarness/Program.cs +++ b/perf/ServerLoadHarness/Program.cs @@ -500,7 +500,7 @@ private static (double AllocatedMb, double CpuMs) ParseProcessReport(string repo } /// - /// Process-wide counters of the server process, captured at the start and end of the window. + /// Process-wide counters, captured at the start and end of the measurement window. /// internal sealed record ProcessStats( long Allocated, @@ -748,7 +748,6 @@ public SubscribeWorkload( m_publishingInterval = publishingInterval; m_writeInterval = writeInterval; m_itemCallback = itemCallback; - m_handler = new NotificationHandler(this); } public override double Operations => Interlocked.Read(ref m_notifications); @@ -897,8 +896,9 @@ private void RecordValue(long now, DataValue value) private async Task CreateManagedSubscriptionAsync(ManagedSession session, CancellationToken ct) { + var setup = new SubscriptionSetup(); V2.ISubscription subscription = session.AddSubscription( - m_handler, + new NotificationHandler(this, setup), new V2.SubscriptionOptions { PublishingInterval = TimeSpan.FromMilliseconds(m_publishingInterval), @@ -908,7 +908,7 @@ private async Task CreateManagedSubscriptionAsync(ManagedSession session, Cancel }); for (int i = 0; i < m_nodeIds.Count; i++) { - subscription.TryAddMonitoredItem( + if (!subscription.TryAddMonitoredItem( i.ToString(CultureInfo.InvariantCulture), new V2.MonitoredItems.MonitoredItemOptions { @@ -918,12 +918,13 @@ private async Task CreateManagedSubscriptionAsync(ManagedSession session, Cancel SamplingInterval = TimeSpan.Zero, QueueSize = 1 }, - out _); - } - while (!subscription.Created || subscription.MonitoredItems.Items.Any(item => !item.Created)) - { - await Task.Delay(50, ct).ConfigureAwait(false); + out _)) + { + throw new InvalidOperationException( + $"Could not add monitored item '{m_nodeIds[i]}' during subscription setup."); + } } + await setup.WaitForReadyAsync(subscription, ct).ConfigureAwait(false); } /// @@ -931,9 +932,10 @@ private async Task CreateManagedSubscriptionAsync(ManagedSession session, Cancel /// private sealed class NotificationHandler : V2.ISubscriptionNotificationHandler { - public NotificationHandler(SubscribeWorkload owner) + public NotificationHandler(SubscribeWorkload owner, SubscriptionSetup setup) { m_owner = owner; + m_setup = setup; } public ValueTask OnDataChangeNotificationAsync( @@ -979,10 +981,12 @@ public ValueTask OnSubscriptionStateChangedAsync( V2.PublishState publishStateMask, CancellationToken ct = default) { + m_setup.OnStateChanged(state); return ValueTask.CompletedTask; } private readonly SubscribeWorkload m_owner; + private readonly SubscriptionSetup m_setup; } private readonly ISession[] m_sessions; @@ -991,7 +995,6 @@ public ValueTask OnSubscriptionStateChangedAsync( private readonly int m_publishingInterval; private readonly int m_writeInterval; private readonly bool m_itemCallback; - private readonly NotificationHandler m_handler; private readonly long[] m_writeTicks = new long[1 << 16]; private readonly LatencyRecorder m_latency = new(); private long m_notifications; diff --git a/perf/ServerLoadHarness/SubscriptionSetup.cs b/perf/ServerLoadHarness/SubscriptionSetup.cs new file mode 100644 index 0000000000..55e07c92cc --- /dev/null +++ b/perf/ServerLoadHarness/SubscriptionSetup.cs @@ -0,0 +1,116 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Client.Subscriptions; +using Opc.Ua.Client.Subscriptions.MonitoredItems; + +namespace Opc.Ua.Perf.ServerLoadHarness +{ + /// + /// Waits for one managed subscription and its monitored items to be ready for measurement. + /// + internal sealed class SubscriptionSetup + { + /// + /// Creates a setup observer using the supplied clock or the system clock. + /// + internal SubscriptionSetup(TimeProvider? timeProvider = null) + { + m_timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + /// Retains the first subscription failure, including failures reported before waiting starts. + /// + internal void OnStateChanged(SubscriptionState state) + { + if (state is SubscriptionState.Error or SubscriptionState.Deleted) + { + Interlocked.CompareExchange(ref m_failureState, (int)state, 0); + } + } + + /// + /// Waits up to one minute for the subscription and all of its items to be created. + /// + /// A monitored item was rejected. + /// The subscription failed or was deleted. + /// Setup did not complete before the deadline. + /// The caller cancelled setup. + internal async Task WaitForReadyAsync(ISubscription subscription, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + using var timeout = m_timeProvider.CreateCancellationTokenSource(TimeSpan.FromMinutes(1)); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct, timeout.Token); + try + { + while (true) + { + stop.Token.ThrowIfCancellationRequested(); + var state = (SubscriptionState)Volatile.Read(ref m_failureState); + if (state is SubscriptionState.Error or SubscriptionState.Deleted) + { + throw new InvalidOperationException( + $"Managed subscription setup failed with state {state}."); + } + + bool ready = subscription.Created; + foreach (IMonitoredItem item in subscription.MonitoredItems.Items) + { + ServiceResult error = item.Error; + if (ServiceResult.IsBad(error)) + { + throw new ServiceResultException( + error.StatusCode, + $"Monitored item '{item.Name}' failed during subscription setup: {error}"); + } + ready &= item.Created; + } + if (ready) + { + return; + } + + await m_timeProvider.Delay(TimeSpan.FromMilliseconds(50), stop.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException exception) when (timeout.IsCancellationRequested && + !ct.IsCancellationRequested) + { + throw new TimeoutException("Managed subscription setup did not complete within one minute.", exception); + } + } + + private readonly TimeProvider m_timeProvider; + private int m_failureState; + } +} diff --git a/src/Opc.Ua.Core/Security/Certificates/AesCache.cs b/src/Opc.Ua.Core/Security/Certificates/AesCache.cs new file mode 100644 index 0000000000..0f4660d7e4 --- /dev/null +++ b/src/Opc.Ua.Core/Security/Certificates/AesCache.cs @@ -0,0 +1,82 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Security.Cryptography; +using System.Threading; + +namespace Opc.Ua +{ + /// + /// Retains one keyed AES instance without sharing it between concurrent transforms. + /// Borrowed instances are disposed on return if shutdown has begun. + /// + internal sealed class AesCache : IDisposable + { + /// + /// Closes the cache and disposes the instance not currently owned by a transform. + /// + public void Dispose() + { + Volatile.Write(ref m_disposed, true); + Interlocked.Exchange(ref m_cached, null)?.Dispose(); + } + + /// + /// Takes exclusive ownership of the cached instance, if any. + /// + internal Aes? Take() + { + return Interlocked.Exchange(ref m_cached, null); + } + + /// + /// Returns an instance, disposing any surplus instance or a return after shutdown. + /// + internal void Return(Aes aes) + { + if (Volatile.Read(ref m_disposed)) + { + aes.Dispose(); + return; + } + + Interlocked.Exchange(ref m_cached, aes)?.Dispose(); + + // Disposal may have drained the cache between the first check and the exchange. + if (Volatile.Read(ref m_disposed)) + { + Interlocked.Exchange(ref m_cached, null)?.Dispose(); + } + } + + private Aes? m_cached; + private bool m_disposed; + } +} diff --git a/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs b/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs index 4510db6a49..822734bcb6 100644 --- a/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs +++ b/src/Opc.Ua.Core/Security/Certificates/CryptoUtils.cs @@ -902,26 +902,18 @@ public static ArraySegment SymmetricEncryptAndSign( uint lastSequenceNumber, ISymmetricCryptoProvider? provider) { - Aes? aes = null; - try - { - return SymmetricEncryptAndSign( - data, - securityPolicy, - encryptingKey, - iv, - signingKey, - hmac, - signOnly, - tokenId, - lastSequenceNumber, - provider, - ref aes); - } - finally - { - aes?.Dispose(); - } + return SymmetricEncryptAndSign( + data, + securityPolicy, + encryptingKey, + iv, + signingKey, + hmac, + signOnly, + tokenId, + lastSequenceNumber, + provider, + null); } /// @@ -929,10 +921,9 @@ public static ArraySegment SymmetricEncryptAndSign( /// for the CBC policies. /// /// - /// aesCache is the AES instance for kept by the - /// caller (one per channel token and direction), or - /// to have one created and stored there. Creating an instance imports the - /// key, which costs more than encrypting a small message. + /// aesCache retains an AES instance for + /// (one per channel token and direction), or is to use a fresh instance. + /// Creating an instance imports the key, which costs more than encrypting a small message. /// /// /// @@ -947,7 +938,7 @@ internal static ArraySegment SymmetricEncryptAndSign( uint tokenId, uint lastSequenceNumber, ISymmetricCryptoProvider? provider, - ref Aes? aesCache) + AesCache? aesCache) { SymmetricEncryptionAlgorithm algorithm = securityPolicy.SymmetricEncryptionAlgorithm; @@ -1070,7 +1061,7 @@ internal static ArraySegment SymmetricEncryptAndSign( dataArray, data.Offset, data.Count, - ref aesCache); + aesCache); } } @@ -1611,26 +1602,18 @@ public static ArraySegment SymmetricDecryptAndVerify( HMAC? hmac, ISymmetricCryptoProvider? provider) { - Aes? aes = null; - try - { - return SymmetricDecryptAndVerify( - data, - securityPolicy, - encryptingKey, - iv, - signingKey, - signOnly, - tokenId, - lastSequenceNumber, - hmac, - provider, - ref aes); - } - finally - { - aes?.Dispose(); - } + return SymmetricDecryptAndVerify( + data, + securityPolicy, + encryptingKey, + iv, + signingKey, + signOnly, + tokenId, + lastSequenceNumber, + hmac, + provider, + null); } /// @@ -1638,9 +1621,8 @@ public static ArraySegment SymmetricDecryptAndVerify( /// AES instance for the CBC policies. /// /// - /// aesCache is the AES instance for kept by the - /// caller (one per channel token and direction), or - /// to have one created and stored there. + /// aesCache retains an AES instance for + /// (one per channel token and direction), or is to use a fresh instance. /// /// /// @@ -1658,7 +1640,7 @@ internal static ArraySegment SymmetricDecryptAndVerify( uint lastSequenceNumber, HMAC? hmac, ISymmetricCryptoProvider? provider, - ref Aes? aesCache) + AesCache? aesCache) { SymmetricEncryptionAlgorithm algorithm = securityPolicy.SymmetricEncryptionAlgorithm; @@ -1727,7 +1709,7 @@ internal static ArraySegment SymmetricDecryptAndVerify( dataArray, data.Offset, data.Count, - ref aesCache); + aesCache); } } @@ -1843,9 +1825,9 @@ private static void AesCbcTransform( byte[] buffer, int offset, int count, - ref Aes? aesCache) + AesCache? aesCache) { - Aes aes = Interlocked.Exchange(ref aesCache, null) ?? CreateCbcAes(key); + Aes aes = aesCache?.Take() ?? CreateCbcAes(key); try { #if NET6_0_OR_GREATER @@ -1870,8 +1852,14 @@ private static void AesCbcTransform( } finally { - // An instance another caller returned meanwhile is surplus. - Interlocked.Exchange(ref aesCache, aes)?.Dispose(); + if (aesCache == null) + { + aes.Dispose(); + } + else + { + aesCache.Return(aes); + } } } diff --git a/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs b/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs index 0d8ab05caa..97ff7059a8 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/ChannelToken.cs @@ -64,8 +64,8 @@ private void Dispose(bool disposing) ClientHmac?.Dispose(); ClientHmac = null; - Interlocked.Exchange(ref ServerAes, null)?.Dispose(); - Interlocked.Exchange(ref ClientAes, null)?.Dispose(); + ServerAes.Dispose(); + ClientAes.Dispose(); m_localNonce?.Dispose(); m_localNonce = null; @@ -216,17 +216,14 @@ internal bool IsActivationRequired(TimeProvider timeProvider) internal HMAC? ClientHmac { get; set; } /// - /// The AES instance for with the CBC - /// policies, created on first use. A field so that it can be taken and - /// returned atomically. + /// The AES cache for with the CBC policies. /// - internal Aes? ServerAes; + internal AesCache ServerAes { get; } = new(); /// - /// The AES instance for with the CBC - /// policies, created on first use. + /// The AES cache for with the CBC policies. /// - internal Aes? ClientAes; + internal AesCache ClientAes { get; } = new(); /// /// Takes ownership of the actual key-agreement objects while a reconnect is handed off. diff --git a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs index 964dcdef37..3c15f1c82e 100644 --- a/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs +++ b/src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryChannel.Symmetric.cs @@ -782,7 +782,7 @@ private ArraySegment EncryptAndSign( token.TokenId, LastSentSequenceNumber, // already incremented to create this message. need the last one sent. m_symmetricProvider, - ref useClientKeys ? ref token.ClientAes : ref token.ServerAes); + useClientKeys ? token.ClientAes : token.ServerAes); } /// @@ -967,7 +967,7 @@ private ArraySegment DecryptAndVerify( m_remoteSequenceNumber, useClientKeys ? token.ClientHmac : token.ServerHmac, m_symmetricProvider, - ref useClientKeys ? ref token.ClientAes : ref token.ServerAes); + useClientKeys ? token.ClientAes : token.ServerAes); } private static readonly byte[] s_hkdfClientLabel = Encoding.UTF8.GetBytes("opcua-client"); diff --git a/tests/Opc.Ua.Client.Tests/Opc.Ua.Client.Tests.csproj b/tests/Opc.Ua.Client.Tests/Opc.Ua.Client.Tests.csproj index f8eb19a407..3690cc33ef 100644 --- a/tests/Opc.Ua.Client.Tests/Opc.Ua.Client.Tests.csproj +++ b/tests/Opc.Ua.Client.Tests/Opc.Ua.Client.Tests.csproj @@ -31,6 +31,7 @@ + diff --git a/tests/Opc.Ua.Client.Tests/Perf/SubscriptionSetupTests.cs b/tests/Opc.Ua.Client.Tests/Perf/SubscriptionSetupTests.cs new file mode 100644 index 0000000000..1ad625192c --- /dev/null +++ b/tests/Opc.Ua.Client.Tests/Perf/SubscriptionSetupTests.cs @@ -0,0 +1,200 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Moq; +using NUnit.Framework; +using Opc.Ua.Client.Subscriptions.MonitoredItems; +using Opc.Ua.Perf.ServerLoadHarness; +using ManagedSubscription = Opc.Ua.Client.Subscriptions.ISubscription; +using ManagedSubscriptionState = Opc.Ua.Client.Subscriptions.SubscriptionState; + +namespace Opc.Ua.Client.Tests.Perf +{ + /// + /// Exercises the readiness loop used by the managed client load harness. + /// + [TestFixture] + [Parallelizable(ParallelScope.All)] + public sealed class SubscriptionSetupTests + { + /// + /// A ready subscription does not wait for the polling interval. + /// + [Test] + public async Task ReadySubscriptionCompletesImmediatelyAsync() + { + var setup = new SubscriptionSetup(); + Mock subscription = CreateSubscription(true, CreateItem(true).Object); + + Task wait = setup.WaitForReadyAsync(subscription.Object, CancellationToken.None); + + Assert.That(wait.Status, Is.EqualTo(TaskStatus.RanToCompletion)); + await wait.ConfigureAwait(false); + } + + /// + /// Readiness requires both the subscription and every monitored item. + /// + [Test] + public async Task WaitsForTheSubscriptionAndAllItemsAsync() + { + var clock = new FakeTimeProvider(); + var setup = new SubscriptionSetup(clock); + Mock item = CreateItem(false); + Mock subscription = CreateSubscription(false, item.Object); + using var guard = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + Task wait = setup.WaitForReadyAsync(subscription.Object, guard.Token); + Assert.That(wait.IsCompleted, Is.False); + subscription.SetupGet(value => value.Created).Returns(true); + Assert.That(wait.IsCompleted, Is.False); + item.SetupGet(value => value.Created).Returns(true); + clock.Advance(TimeSpan.FromMilliseconds(50)); + + await wait.ConfigureAwait(false); + } + + /// + /// A bad item must fail setup, even when another item is pending or the bad item is created. + /// + [TestCase(false)] + [TestCase(true)] + public void RejectedMonitoredItemFailsSetup(bool created) + { + var setup = new SubscriptionSetup(); + Mock rejected = CreateItem(created); + rejected.SetupGet(value => value.Error).Returns(new ServiceResult(StatusCodes.BadNodeIdUnknown)); + Mock subscription = CreateSubscription( + true, CreateItem(false).Object, rejected.Object); + using var guard = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + + ServiceResultException exception = Assert.ThrowsAsync( + () => setup.WaitForReadyAsync(subscription.Object, guard.Token))!; + + Assert.That(exception.StatusCode, Is.EqualTo((uint)StatusCodes.BadNodeIdUnknown)); + Assert.That(exception.Message, Does.Contain("item")); + } + + /// + /// Error and deletion state notifications fail setup instead of waiting for creation. + /// + [TestCase(ManagedSubscriptionState.Error)] + [TestCase(ManagedSubscriptionState.Deleted)] + public void FailedSubscriptionStateFailsSetup(ManagedSubscriptionState state) + { + var setup = new SubscriptionSetup(); + Mock subscription = CreateSubscription(false, CreateItem(false).Object); + setup.OnStateChanged(state); + using var guard = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + + InvalidOperationException exception = Assert.ThrowsAsync( + () => setup.WaitForReadyAsync(subscription.Object, guard.Token))!; + + Assert.That(exception.Message, Does.Contain(state.ToString())); + } + + /// + /// An error delivered while polling remains visible even if the engine starts recovering. + /// + [Test] + public void ErrorDuringSetupIsNotLostDuringRecovery() + { + var clock = new FakeTimeProvider(); + var setup = new SubscriptionSetup(clock); + Mock subscription = CreateSubscription(false, CreateItem(false).Object); + using var guard = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + Task wait = setup.WaitForReadyAsync(subscription.Object, guard.Token); + + setup.OnStateChanged(ManagedSubscriptionState.Error); + setup.OnStateChanged(ManagedSubscriptionState.Opened); + clock.Advance(TimeSpan.FromMilliseconds(50)); + + Assert.ThrowsAsync(() => wait); + } + + /// + /// Neither missing subscription creation nor missing item creation can wait indefinitely. + /// + [TestCase(false)] + [TestCase(true)] + public void SetupHasAFiniteDeadline(bool subscriptionCreated) + { + var clock = new FakeTimeProvider(); + var setup = new SubscriptionSetup(clock); + Mock subscription = CreateSubscription( + subscriptionCreated, CreateItem(false).Object); + using var guard = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + Task wait = setup.WaitForReadyAsync(subscription.Object, guard.Token); + + clock.Advance(TimeSpan.FromMinutes(1)); + + TimeoutException exception = Assert.ThrowsAsync(() => wait)!; + Assert.That(exception.Message, Does.Contain("subscription")); + } + + /// + /// Caller cancellation is preserved, including when the subscription is already ready. + /// + [TestCase(false)] + [TestCase(true)] + public void CallerCancellationInterruptsSetup(bool ready) + { + var setup = new SubscriptionSetup(); + Mock subscription = CreateSubscription(ready, CreateItem(ready).Object); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.CatchAsync( + () => setup.WaitForReadyAsync(subscription.Object, cancellation.Token)); + } + + private static Mock CreateSubscription(bool created, params IMonitoredItem[] items) + { + var collection = new Mock(); + collection.SetupGet(value => value.Items).Returns(items); + var subscription = new Mock(); + subscription.SetupGet(value => value.Created).Returns(created); + subscription.SetupGet(value => value.MonitoredItems).Returns(collection.Object); + return subscription; + } + + private static Mock CreateItem(bool created) + { + var item = new Mock(); + item.SetupGet(value => value.Name).Returns("item"); + item.SetupGet(value => value.Created).Returns(created); + item.SetupGet(value => value.Error).Returns(ServiceResult.Good); + return item; + } + } +} diff --git a/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs b/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs index 5e47a40c42..e28b92c789 100644 --- a/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs +++ b/tests/Opc.Ua.Core.Tests/Security/Certificates/CryptoUtilsAesCacheTests.cs @@ -42,7 +42,7 @@ namespace Opc.Ua.Core.Tests.Security.Certificates [Category("CryptoUtils")] [Parallelizable] [SetCulture("en-us")] - public class CryptoUtilsAesCacheTests + public sealed class CryptoUtilsAesCacheTests { private const int kHeaderSize = 24; private const int kBlockSize = 16; @@ -55,24 +55,22 @@ public class CryptoUtilsAesCacheTests public void CachedInstanceEncryptsLikeAFreshOne() { byte[] key = CreateBytes(32, 1); - Aes? cache = null; - try + using var cache = new AesCache(); + Aes? firstInstance = null; + for (int ii = 0; ii < 3; ii++) { - for (int ii = 0; ii < 3; ii++) - { - byte[] iv = CreateBytes(kBlockSize, (byte)(10 + ii)); - byte[] body = CreateBytes(37 + ii, (byte)(50 + ii)); + byte[] iv = CreateBytes(kBlockSize, (byte)(10 + ii)); + byte[] body = CreateBytes(37 + ii, (byte)(50 + ii)); - byte[] expected = EncryptWithoutCache(body, key, iv); - byte[] actual = Encrypt(body, key, iv, ref cache); + byte[] expected = EncryptWithoutCache(body, key, iv); + byte[] actual = Encrypt(body, key, iv, cache); - Assert.That(actual, Is.EqualTo(expected), $"message {ii}"); - Assert.That(cache, Is.Not.Null); - } - } - finally - { - cache?.Dispose(); + Assert.That(actual, Is.EqualTo(expected), $"message {ii}"); + Aes? instance = cache.Take(); + Assert.That(instance, Is.Not.Null); + firstInstance ??= instance; + Assert.That(instance, Is.SameAs(firstInstance)); + cache.Return(instance!); } } @@ -84,52 +82,44 @@ public void CachedInstancesRoundTripManyMessages() { byte[] key = CreateBytes(32, 3); byte[] iv = CreateBytes(kBlockSize, 4); - Aes? encryptCache = null; - Aes? decryptCache = null; - try + using var encryptCache = new AesCache(); + using var decryptCache = new AesCache(); + for (int bodySize = 0; bodySize < 40; bodySize++) { - for (int bodySize = 0; bodySize < 40; bodySize++) - { - byte[] body = CreateBytes(bodySize, (byte)bodySize); - byte[] buffer = new byte[512]; - Array.Copy(body, 0, buffer, kHeaderSize, bodySize); - - ArraySegment encrypted = CryptoUtils.SymmetricEncryptAndSign( - new ArraySegment(buffer, kHeaderSize, bodySize), - SecurityPolicyInfo.Basic256Sha256, - key, - iv, - null, - null, - false, - 0, - 0, - null, - ref encryptCache); - - ArraySegment decrypted = CryptoUtils.SymmetricDecryptAndVerify( - new ArraySegment(buffer, kHeaderSize, encrypted.Count - kHeaderSize), - SecurityPolicyInfo.Basic256Sha256, - key, - iv, - null, - false, - 0, - 0, - null, - null, - ref decryptCache); - - Assert.That(decrypted, Has.Count.EqualTo(kHeaderSize + bodySize)); - byte[] roundTripped = new byte[bodySize]; - Array.Copy(buffer, kHeaderSize, roundTripped, 0, bodySize); - Assert.That(roundTripped, Is.EqualTo(body)); - } - } - finally - { - encryptCache?.Dispose(); - decryptCache?.Dispose(); + byte[] body = CreateBytes(bodySize, (byte)bodySize); + byte[] buffer = new byte[512]; + Array.Copy(body, 0, buffer, kHeaderSize, bodySize); + + ArraySegment encrypted = CryptoUtils.SymmetricEncryptAndSign( + new ArraySegment(buffer, kHeaderSize, bodySize), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + null, + false, + 0, + 0, + null, + encryptCache); + + ArraySegment decrypted = CryptoUtils.SymmetricDecryptAndVerify( + new ArraySegment(buffer, kHeaderSize, encrypted.Count - kHeaderSize), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + false, + 0, + 0, + null, + null, + decryptCache); + + Assert.That(decrypted, Has.Count.EqualTo(kHeaderSize + bodySize)); + byte[] roundTripped = new byte[bodySize]; + Array.Copy(buffer, kHeaderSize, roundTripped, 0, bodySize); + Assert.That(roundTripped, Is.EqualTo(body)); } } @@ -144,41 +134,111 @@ public void ConcurrentUsersOfOneCacheGetCorrectResults() byte[] iv = CreateBytes(kBlockSize, 6); byte[] body = CreateBytes(300, 7); byte[] expected = EncryptWithoutCache(body, key, iv); - var holder = new CacheHolder(); - try + using var cache = new AesCache(); + Parallel.For(0, 256, _ => { - Parallel.For(0, 256, _ => - { - byte[] actual = Encrypt(body, key, iv, ref holder.Cache); - Assert.That(actual, Is.EqualTo(expected)); - }); - Assert.That(holder.Cache, Is.Not.Null); + byte[] actual = Encrypt(body, key, iv, cache); + Assert.That(actual, Is.EqualTo(expected)); + }); + using Aes? instance = cache.Take(); + Assert.That(instance, Is.Not.Null); + } + + /// + /// Disposing a channel token disposes and clears its cached instances. + /// + [Test] + public void DisposingTheTokenReleasesTheCachedInstances() + { + using var token = new ChannelToken(); + token.ClientAes.Return(Aes.Create()); + token.ServerAes.Return(Aes.Create()); + + token.Dispose(); + + using Aes? client = token.ClientAes.Take(); + using Aes? server = token.ServerAes.Take(); + Assert.That(client, Is.Null); + Assert.That(server, Is.Null); + } + + /// + /// Disposal while a transform owns an instance must not retain that + /// instance when the transform returns it to the token. + /// + [TestCase(true, true)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(false, false)] + public void DisposingTheTokenDuringATransformReleasesTheInstance(bool client, bool encrypt) + { + byte[] key = CreateBytes(32, 8); + byte[] iv = CreateBytes(kBlockSize, 9); + byte[] body = CreateBytes(37, 10); + byte[] encrypted = EncryptWithoutCache(body, key, iv); + using var token = new ChannelToken(); + using var aes = new TransformCallbackAes(key, token.Dispose); + AesCache cache = client ? token.ClientAes : token.ServerAes; + cache.Return(aes); + + if (encrypt) + { + Assert.That(Encrypt(body, key, iv, cache), Is.EqualTo(encrypted)); } - finally + else { - holder.Cache?.Dispose(); + ArraySegment decrypted = CryptoUtils.SymmetricDecryptAndVerify( + new ArraySegment(encrypted, kHeaderSize, encrypted.Length - kHeaderSize), + SecurityPolicyInfo.Basic256Sha256, + key, + iv, + null, + false, + 0, + 0, + null, + null, + cache); + Assert.That(decrypted, Has.Count.EqualTo(kHeaderSize + body.Length)); + Assert.That(encrypted.AsSpan(kHeaderSize, body.Length).ToArray(), Is.EqualTo(body)); } + + using Aes? clientInstance = token.ClientAes.Take(); + using Aes? serverInstance = token.ServerAes.Take(); + Assert.Multiple(() => + { + Assert.That(clientInstance, Is.Null); + Assert.That(serverInstance, Is.Null); + Assert.That(aes.IsDisposed, Is.True); + }); } /// - /// Disposing a channel token disposes and clears its cached instances. + /// A return racing shutdown must release both the retained and the returned instance. /// [Test] - public void DisposingTheTokenReleasesTheCachedInstances() + public async Task ConcurrentReturnAndDisposalReleaseEveryInstanceAsync() { - var token = new ChannelToken + byte[] key = CreateBytes(32, 11); + for (int i = 0; i < 200; i++) { - ClientAes = Aes.Create(), - ServerAes = Aes.Create() - }; + using var cache = new AesCache(); + using var first = new TransformCallbackAes(key, static () => { }); + using var second = new TransformCallbackAes(key, static () => { }); + cache.Return(first); - token.Dispose(); + await Task.WhenAll( + Task.Run(cache.Dispose), + Task.Run(() => cache.Return(second))).ConfigureAwait(false); - Assert.That(token.ClientAes, Is.Null); - Assert.That(token.ServerAes, Is.Null); + using Aes? instance = cache.Take(); + Assert.That(instance, Is.Null); + Assert.That(first.IsDisposed, Is.True); + Assert.That(second.IsDisposed, Is.True); + } } - private static byte[] Encrypt(byte[] body, byte[] key, byte[] iv, ref Aes? cache) + private static byte[] Encrypt(byte[] body, byte[] key, byte[] iv, AesCache cache) { byte[] buffer = new byte[kHeaderSize + body.Length + (2 * kBlockSize)]; Array.Copy(body, 0, buffer, kHeaderSize, body.Length); @@ -193,7 +253,7 @@ private static byte[] Encrypt(byte[] body, byte[] key, byte[] iv, ref Aes? cache 0, 0, null, - ref cache); + cache); byte[] result = new byte[encrypted.Count]; Array.Copy(buffer, 0, result, 0, encrypted.Count); return result; @@ -223,9 +283,99 @@ private static byte[] CreateBytes(int length, byte seed) return bytes; } - private sealed class CacheHolder + /// + /// Runs a callback after the real transform has borrowed the cached AES instance. + /// + private sealed class TransformCallbackAes : Aes { - public Aes? Cache; + /// + /// Creates a CBC cipher with a callback at the transform boundary. + /// + public TransformCallbackAes(byte[] key, Action onTransform) + { + m_aes = Aes.Create(); + m_aes.Key = key; + m_aes.Mode = CipherMode.CBC; + m_aes.Padding = PaddingMode.None; + Key = key; + Padding = PaddingMode.None; + m_onTransform = onTransform; + } + + /// + /// Whether the cache disposed this instance. + /// + public bool IsDisposed { get; private set; } + + /// + public override ICryptoTransform CreateEncryptor(byte[] rgbKey, byte[]? rgbIV) + { + m_onTransform(); +#pragma warning disable CA5401 // Test-only forwarding of the protocol IV. TODO: remove when net48 is retired. + return m_aes.CreateEncryptor(rgbKey, rgbIV); +#pragma warning restore CA5401 + } + + /// + public override ICryptoTransform CreateDecryptor(byte[] rgbKey, byte[]? rgbIV) + { + m_onTransform(); + return m_aes.CreateDecryptor(rgbKey, rgbIV); + } + + /// + public override void GenerateIV() + { + m_aes.GenerateIV(); + IV = m_aes.IV; + } + + /// + public override void GenerateKey() + { + m_aes.GenerateKey(); + Key = m_aes.Key; + } + +#if NET6_0_OR_GREATER + /// + protected override bool TryEncryptCbcCore( + ReadOnlySpan plaintext, + ReadOnlySpan iv, + Span destination, + PaddingMode paddingMode, + out int bytesWritten) + { + m_onTransform(); + return m_aes.TryEncryptCbc(plaintext, iv, destination, out bytesWritten, paddingMode); + } + + /// + protected override bool TryDecryptCbcCore( + ReadOnlySpan ciphertext, + ReadOnlySpan iv, + Span destination, + PaddingMode paddingMode, + out int bytesWritten) + { + m_onTransform(); + return m_aes.TryDecryptCbc(ciphertext, iv, destination, out bytesWritten, paddingMode); + } +#endif + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + IsDisposed = true; + m_aes.Dispose(); + } + base.Dispose(disposing); + } + + private readonly Aes m_aes; + private readonly Action m_onTransform; } } } diff --git a/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs index 6ca7c1bb6e..ef1281057b 100644 --- a/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs +++ b/tests/Opc.Ua.Core.Tests/Stack/Transport/ChannelAsyncOperationWaitTests.cs @@ -157,14 +157,15 @@ public async Task TheContinuationDoesNotRunOnTheCompletingThreadAsync() int continuationThread = -1; using var completed = new ManualResetEventSlim(); - Task waiter = Task.Run(async () => + async Task WaitAndRecordContinuationAsync() { await operation.WaitForCompletionAsync(CancellationToken.None).ConfigureAwait(false); continuationThread = Environment.CurrentManagedThreadId; - }); + } - // let the waiter register before completing on a dedicated thread. - await Task.Delay(50).ConfigureAwait(false); + // Returning from the direct call guarantees that the pending continuation is registered. + Task waiter = WaitAndRecordContinuationAsync(); + Assert.That(waiter.IsCompleted, Is.False); var thread = new Thread(() => { completingThread = Environment.CurrentManagedThreadId; @@ -174,9 +175,15 @@ public async Task TheContinuationDoesNotRunOnTheCompletingThreadAsync() }); thread.Start(); - await waiter.ConfigureAwait(false); - completed.Set(); - thread.Join(); + try + { + await waiter.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); + } + finally + { + completed.Set(); + Assert.That(thread.Join(TimeSpan.FromSeconds(5)), Is.True); + } Assert.That(continuationThread, Is.Not.EqualTo(completingThread)); } diff --git a/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs b/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs index ece1a3a55d..eaad377e35 100644 --- a/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs +++ b/tests/Opc.Ua.Core.Tests/Stack/Transport/UaSCBinaryChannelWriteQueueTests.cs @@ -253,7 +253,7 @@ public List Sent { get { - lock (m_sent) + lock (m_sentLock) { return new List(m_sent); } @@ -329,7 +329,7 @@ private async ValueTask SendAsync(int index) throw ServiceResultException.Create(StatusCodes.BadConnectionClosed, "send failed"); } - lock (m_sent) + lock (m_sentLock) { m_sent.Add(index); } @@ -340,6 +340,7 @@ private async ValueTask SendAsync(int index) } } + private readonly Lock m_sentLock = new(); private readonly List m_sent = []; private int m_concurrent; private int m_maxConcurrent;