Fix the Sessions LoadTest fixture and the server bottlenecks it exposed

src/Opc.Ua.Core/Stack/Tcp/UaSCBinaryClientChannel.cs

@@ -367,7 +367,11 @@ public async ValueTask<IServiceResponse> SendRequestAsync(
 
             using Activity? activity = CachedActivitySource.StartActivity();
             WriteOperation? operation = null;
-            using (Gate.Enter())
+
+            // a request that waits for the gate (another request or a response is being
+            // processed) must not block its thread: a blocking wait spins first, and with
+            // many concurrent requests on one channel the spinning took a fifth of the CPU.
+            using (await Gate.EnterAsync(ct).ConfigureAwait(false))
             {
                 // Queue the operation while connecting and it will be played once connected.
                 if (State == TcpChannelState.Connecting)

src/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs

@@ -10602,13 +10602,24 @@ private async ValueTask SetDefaultPermissionsAsync(
             NodeState target,
             CancellationToken cancellationToken = default)
         {
+            ushort namespaceIndex = target.NodeId.NamespaceIndex;
+            OperationContext? operationContext = systemContext.OperationContext;
+            if (operationContext != null &&
+                operationContext.TryGetDefaultPermissions(namespaceIndex, out NamespaceDefaultPermissions cached))
+            {
+                return (
+                    cached.DefaultAccessRestrictions,
+                    cached.DefaultRolePermissions,
+                    cached.DefaultUserRolePermissions);
+            }
+
             AccessRestrictionType defaultAccessRestrictions = AccessRestrictionType.None;
             ArrayOf<RolePermissionType> defaultRolePermissions = default;
             ArrayOf<RolePermissionType> defaultUserRolePermissions = default;
 
             // check if NamespaceMetadata is defined for NamespaceIndex of the node.
             NamespaceMetadataState? namespaceMetadataState = await
-                Server.NodeManager.ConfigurationNodeManager!.GetNamespaceMetadataStateAsync(target.NodeId.NamespaceIndex, cancellationToken)
+                Server.NodeManager.ConfigurationNodeManager!.GetNamespaceMetadataStateAsync(namespaceIndex, cancellationToken)
                 .ConfigureAwait(false);
 
             if (namespaceMetadataState != null)
@@ -10666,6 +10677,12 @@ private async ValueTask SetDefaultPermissionsAsync(
                 }
             }
 
+            operationContext?.SetDefaultPermissions(new NamespaceDefaultPermissions(
+                namespaceIndex,
+                defaultAccessRestrictions,
+                defaultRolePermissions,
+                defaultUserRolePermissions));
+
             return (defaultAccessRestrictions, defaultRolePermissions, defaultUserRolePermissions);
         }
 

src/Opc.Ua.Server/NodeManager/CustomNodeManager.cs

@@ -6332,6 +6332,37 @@ private void SetDefaultPermissions(
             ServerSystemContext systemContext,
             NodeState target,
             NodeMetadata metadata)
+        {
+            // the defaults are read once per namespace and service request, like
+            // AsyncCustomNodeManager does: every read takes two server-wide locks and
+            // copies the role permission array.
+            ushort namespaceIndex = target.NodeId.NamespaceIndex;
+            OperationContext? operationContext = systemContext.OperationContext;
+            if (operationContext != null &&
+                operationContext.TryGetDefaultPermissions(namespaceIndex, out NamespaceDefaultPermissions cached))
+            {
+                metadata.DefaultAccessRestrictions = cached.DefaultAccessRestrictions;
+                metadata.DefaultRolePermissions = cached.DefaultRolePermissions;
+                metadata.DefaultUserRolePermissions = cached.DefaultUserRolePermissions;
+                return;
+            }
+
+            ReadDefaultPermissions(systemContext, target, metadata);
+
+            operationContext?.SetDefaultPermissions(new NamespaceDefaultPermissions(
+                namespaceIndex,
+                metadata.DefaultAccessRestrictions,
+                metadata.DefaultRolePermissions,
+                metadata.DefaultUserRolePermissions));
+        }
+
+        /// <summary>
+        /// Reads the namespace default permissions of the node's namespace into the metadata.
+        /// </summary>
+        private void ReadDefaultPermissions(
+            ServerSystemContext systemContext,
+            NodeState target,
+            NodeMetadata metadata)
         {
             // check if NamespaceMetadata is defined for NamespaceUri
             string? namespaceUri = Server.NamespaceUris.GetString(target.NodeId.NamespaceIndex);

src/Opc.Ua.Server/NodeManager/MasterNodeManager.cs

@@ -1613,7 +1613,8 @@ void ISyncNodeManagerMonitoredItemRecovery.RecoverDetachedMonitoredItems(
             IAsyncNodeManager nodeManager,
             IReadOnlyCollection<NodeId> nodeIds)
         {
-            if (!Server.IsRunning)
+            // only items that are detached or marked deleted are recovered.
+            if (!Server.IsRunning || !MonitoredItem.HasRecoverableItems)
             {
                 return;
             }
@@ -1684,7 +1685,8 @@ async ValueTask IDynamicNodeManagerHost.RecoverDetachedMonitoredItemsAsync(
             IReadOnlyCollection<NodeId>? nodeIds,
             CancellationToken cancellationToken)
         {
-            if (!Server.IsRunning)
+            // only items that are detached or marked deleted are recovered.
+            if (!Server.IsRunning || !MonitoredItem.HasRecoverableItems)
             {
                 return;
             }

src/Opc.Ua.Server/NodeManager/ResourceManager.cs

@@ -316,11 +316,14 @@ public void Add(StatusCode statusCode, string locale, string text)
 
                 Add(key, locale, text);
 
-                m_statusCodeMapping ??= [];
-
                 if (string.IsNullOrEmpty(locale) || locale == "en-US")
                 {
-                    m_statusCodeMapping[statusCode] = new TranslationInfo(key, locale, text);
+                    // copy on write: lookups read the map without the lock.
+                    Dictionary<StatusCode, TranslationInfo> mapping = m_statusCodeMapping != null
+                        ? new Dictionary<StatusCode, TranslationInfo>(m_statusCodeMapping)
+                        : [];
+                    mapping[statusCode] = new TranslationInfo(key, locale, text);
+                    Volatile.Write(ref m_statusCodeMapping, mapping);
                 }
             }
         }
@@ -338,11 +341,14 @@ public void Add(XmlQualifiedName symbolicId, string locale, string text)
 
                     Add(key, locale, text);
 
-                    m_symbolicIdMapping ??= [];
-
                     if (string.IsNullOrEmpty(locale) || locale == "en-US")
                     {
-                        m_symbolicIdMapping[symbolicId] = new TranslationInfo(key, locale, text);
+                        // copy on write: lookups read the map without the lock.
+                        Dictionary<XmlQualifiedName, TranslationInfo> mapping = m_symbolicIdMapping != null
+                            ? new Dictionary<XmlQualifiedName, TranslationInfo>(m_symbolicIdMapping)
+                            : [];
+                        mapping[symbolicId] = new TranslationInfo(key, locale, text);
+                        Volatile.Write(ref m_symbolicIdMapping, mapping);
                     }
                 }
             }
@@ -511,9 +517,11 @@ public TranslationTable(string locale)
             public string Language { get; }
 
             /// <summary>
-            /// The translations by key.
+            /// The translations by key. Keys are identifiers, compared ordinally: the
+            /// default string comparer is culture-sensitive and made every lookup of a
+            /// Browse display name cost up to a microsecond.
             /// </summary>
-            public SortedDictionary<string, string> Translations { get; init; } = [];
+            public Dictionary<string, string> Translations { get; init; } = new(StringComparer.Ordinal);
         }
 
         /// <summary>
@@ -541,7 +549,9 @@ private TranslationTable[] GetTranslationSnapshot()
                         TranslationTable table = m_translationTables[ii];
                         snapshot[ii] = new TranslationTable(table.Locale)
                         {
-                            Translations = new SortedDictionary<string, string>(table.Translations)
+                            Translations = new Dictionary<string, string>(
+                                table.Translations,
+                                StringComparer.Ordinal)
                         };
                     }
                     Volatile.Write(ref m_translationSnapshot, snapshot);
@@ -959,19 +969,18 @@ private LocalizedText TranslateStatusCode(
             object[] args,
             string? symbolicId = null)
         {
-            lock (m_lock)
+            // the map is replaced, never changed, so it is read without the lock: a session
+            // that requests diagnostics translates the status of every notification.
+            Dictionary<StatusCode, TranslationInfo>? mapping = Volatile.Read(ref m_statusCodeMapping);
+            if (mapping != null && mapping.TryGetValue(statusCode.Code, out TranslationInfo info))
             {
-                if (m_statusCodeMapping != null &&
-                    m_statusCodeMapping.TryGetValue(statusCode.Code, out TranslationInfo info))
+                // merge the argument list with the translation info cached for the status code.
+                if (args != null)
                 {
-                    // merge the argument list with the translation info cached for the status code.
-                    if (args != null)
-                    {
-                        info = new TranslationInfo(info.Key, info.Locale, info.Text, args);
-                    }
-
-                    return Translate(preferredLocales, default, info);
+                    info = new TranslationInfo(info.Key, info.Locale, info.Text, args);
                 }
+
+                return Translate(preferredLocales, default, info);
             }
 
             return LocalizedText.From(symbolicId ?? Utils.Format("{0:X8}", statusCode.Code));
@@ -987,21 +996,17 @@ private LocalizedText TranslateSymbolicId(
             StatusCode statusCode,
             object[] args)
         {
-            lock (m_lock)
+            Dictionary<XmlQualifiedName, TranslationInfo>? mapping = Volatile.Read(ref m_symbolicIdMapping);
+            if (mapping != null &&
+                mapping.TryGetValue(new XmlQualifiedName(symbolicId, namespaceUri), out TranslationInfo info))
             {
-                if (m_symbolicIdMapping != null &&
-                    m_symbolicIdMapping.TryGetValue(
-                        new XmlQualifiedName(symbolicId, namespaceUri),
-                        out TranslationInfo info))
+                // merge the argument list with the translation info cached for the symbolic id.
+                if (args != null)
                 {
-                    // merge the argument list with the translation info cached for the symbolic id.
-                    if (args != null)
-                    {
-                        info = new TranslationInfo(info.Key, info.Locale, info.Text, args);
-                    }
-
-                    return Translate(preferredLocales, default, info);
+                    info = new TranslationInfo(info.Key, info.Locale, info.Text, args);
                 }
+
+                return Translate(preferredLocales, default, info);
             }
 
             if ((string.IsNullOrEmpty(namespaceUri) || namespaceUri == Ua.Namespaces.OpcUa) &&

src/Opc.Ua.Server/Server/OperationContext.cs

@@ -434,9 +434,92 @@ public SystemContextCacheEntry(ServerSystemContext template, ServerSystemContext
             public ServerSystemContext Copy { get; }
         }
 
+        /// <summary>
+        /// Returns the namespace default permissions remembered for this service
+        /// request by <see cref="SetDefaultPermissions"/>.
+        /// </summary>
+        /// <remarks>
+        /// Permission checks read the DefaultAccessRestrictions, DefaultRolePermissions
+        /// and DefaultUserRolePermissions of the node's namespace for every node of a
+        /// request, which took two server-wide locks and deep-copied the role permission
+        /// array per node. One read per namespace and request is enough. Contexts that
+        /// are not a service request (monitored item and session contexts, which can be
+        /// kept for a long time) remember nothing, so a changed default applies to them
+        /// at once.
+        /// </remarks>
+        internal bool TryGetDefaultPermissions(
+            ushort namespaceIndex,
+            out NamespaceDefaultPermissions permissions)
+        {
+            NamespaceDefaultPermissions[]? entries = m_defaultPermissions;
+            if (entries != null)
+            {
+                foreach (NamespaceDefaultPermissions entry in entries)
+                {
+                    if (entry.NamespaceIndex == namespaceIndex)
+                    {
+                        permissions = entry;
+                        return true;
+                    }
+                }
+            }
+
+            permissions = default;
+            return false;
+        }
+
+        /// <summary>
+        /// Remembers the namespace default permissions for the rest of this service request.
+        /// </summary>
+        internal void SetDefaultPermissions(NamespaceDefaultPermissions permissions)
+        {
+            if (RequestType == RequestType.Unknown)
+            {
+                return;
+            }
+
+            // copy on write: node managers can validate nodes of one request concurrently,
+            // and a lost update only costs a second read.
+            NamespaceDefaultPermissions[]? entries = m_defaultPermissions;
+            if (entries == null)
+            {
+                m_defaultPermissions = [permissions];
+                return;
+            }
+
+            var updated = new NamespaceDefaultPermissions[entries.Length + 1];
+            Array.Copy(entries, updated, entries.Length);
+            updated[entries.Length] = permissions;
+            m_defaultPermissions = updated;
+        }
+
         private IDisposable? m_requestScope;
         private IDisposable? m_sessionlessLease;
         private SystemContextCacheEntry? m_systemContextCache;
+        private NamespaceDefaultPermissions[]? m_defaultPermissions;
         private static uint s_lastRequestId;
     }
+
+    /// <summary>
+    /// The default permissions of a namespace, read from its NamespaceMetadata.
+    /// </summary>
+    internal readonly struct NamespaceDefaultPermissions
+    {
+        public NamespaceDefaultPermissions(
+            ushort namespaceIndex,
+            AccessRestrictionType defaultAccessRestrictions,
+            ArrayOf<RolePermissionType> defaultRolePermissions,
+            ArrayOf<RolePermissionType> defaultUserRolePermissions)
+        {
+            NamespaceIndex = namespaceIndex;
+            DefaultAccessRestrictions = defaultAccessRestrictions;
+            DefaultRolePermissions = defaultRolePermissions;
+            DefaultUserRolePermissions = defaultUserRolePermissions;
+        }
+
+        public ushort NamespaceIndex { get; }
+        public AccessRestrictionType DefaultAccessRestrictions { get; }
+        public ArrayOf<RolePermissionType> DefaultRolePermissions { get; }
+        public ArrayOf<RolePermissionType> DefaultUserRolePermissions { get; }
+    }
 }