[Rule based auto tagging] Add in-memory rule processing service

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add dfs transformation function in XContentMapValues ([#17612](https://github.com/opensearch-project/OpenSearch/pull/17612))
 - Added Kinesis support as a plugin for the pull-based ingestion ([#17615](https://github.com/opensearch-project/OpenSearch/pull/17615))
 - Add FilterFieldType for developers who want to wrap MappedFieldType ([#17627](https://github.com/opensearch-project/OpenSearch/pull/17627))
+- [Rule Based Auto-tagging] Add in-memory rule processing service ([#17365](https://github.com/opensearch-project/OpenSearch/pull/17365))
 - [Security Manager Replacement] Create initial Java Agent to intercept Socket::connect calls ([#17724](https://github.com/opensearch-project/OpenSearch/pull/17724))
 - Add ingestion management APIs for pause, resume and get ingestion state ([#17631](https://github.com/opensearch-project/OpenSearch/pull/17631))
 - [Security Manager Replacement] Enhance Java Agent to intercept System::exit ([#17746](https://github.com/opensearch-project/OpenSearch/pull/17746))

libs/autotagging-commons/build.gradle

@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+dependencies {
+  api 'org.apache.commons:commons-collections4:4.4'
+  api project(":server")
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-core'
+  }
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /commons-collections.*/, to: 'commons-collections'
+}

libs/autotagging-commons/src/main/java/org/opensearch/rule/InMemoryRuleProcessingService.java

@@ -0,0 +1,147 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rule;
+
+import org.opensearch.autotagging.Attribute;
+import org.opensearch.autotagging.FeatureType;
+import org.opensearch.autotagging.Rule;
+import org.opensearch.rule.attribute_extractor.AttributeExtractor;
+import org.opensearch.rule.storage.AttributeValueStore;
+import org.opensearch.rule.storage.AttributeValueStoreFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * This class is responsible for managing in-memory view of Rules and Find matching Rule for the request
+ * Each auto-tagging feature should use a separate instance of this class as this avoid potential concurrency overhead
+ * in case of dynamic updates and attribute sharing scenarios
+ */
+public class InMemoryRuleProcessingService {
+
+    private final AttributeValueStoreFactory attributeValueStoreFactory;
+
+    /**
+     *  Constrcutor
+     * @param featureType
+     * @param attributeValueStoreSupplier
+     */
+    public InMemoryRuleProcessingService(
+        FeatureType featureType,
+        Supplier<AttributeValueStore<String, String>> attributeValueStoreSupplier
+    ) {
+        attributeValueStoreFactory = new AttributeValueStoreFactory(featureType, attributeValueStoreSupplier);
+    }
+
+    /**
+     * Adds the rule to in-memory view
+     * @param rule to be added
+     */
+    public synchronized void add(final Rule rule) {
+        new AddRuleOperation(rule, attributeValueStoreFactory).perform();
+    }
+
+    /**
+     * Removes the rule from in-memory view
+     * @param rule to be removed
+     */
+    public synchronized void remove(final Rule rule) {
+        new DeleteRuleOperation(rule, attributeValueStoreFactory).perform();
+    }
+
+    /**
+     * Evaluates the label for the current request. It finds the matches for each attribute value and then it is an
+     * intersection of all the matches
+     * @param attributeExtractors list of extractors which are used to get the attribute values to find the
+     *                           matching rule
+     * @return a label if there is unique label otherwise empty
+     */
+    public Optional<String> evaluateLabel(List<AttributeExtractor<String>> attributeExtractors) {
+        assert attributeValueStoreFactory != null;
+        Optional<String> result = Optional.empty();
+        for (AttributeExtractor<String> attributeExtractor : attributeExtractors) {
+            AttributeValueStore<String, String> valueStore = attributeValueStoreFactory.getAttributeValueStore(
+                attributeExtractor.getAttribute()
+            );
+            for (String value : attributeExtractor.extract()) {
+                Optional<String> possibleMatch = valueStore.get(value);
+
+                if (possibleMatch.isEmpty()) {
+                    return Optional.empty();
+                }
+
+                if (result.isEmpty()) {
+                    result = possibleMatch;
+                } else {
+                    boolean isThePossibleMatchEqualResult = possibleMatch.get().equals(result.get());
+                    if (!isThePossibleMatchEqualResult) {
+                        return Optional.empty();
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
+    private static abstract class RuleProcessingOperation {
+        protected final Rule rule;
+        protected final AttributeValueStoreFactory attributeValueStoreFactory;
+
+        public RuleProcessingOperation(Rule rule, AttributeValueStoreFactory attributeValueStoreFactory) {
+            this.rule = rule;
+            this.attributeValueStoreFactory = attributeValueStoreFactory;
+        }
+
+        void perform() {
+            final FeatureType feature = rule.getFeatureType();
+            final String label = rule.getFeatureValue();
+
+            for (Map.Entry<Attribute, Set<String>> attributeEntry : rule.getAttributeMap().entrySet()) {
+                processAttributeEntry(attributeEntry);
+            }
+        }
+
+        protected AttributeValueStore<String, String> getAttributeValueStore(final Attribute attribute) {
+            return this.attributeValueStoreFactory.getAttributeValueStore(attribute);
+        }
+
+        protected abstract void processAttributeEntry(Map.Entry<Attribute, Set<String>> attributeEntry);
+    }
+
+    private static class DeleteRuleOperation extends RuleProcessingOperation {
+        public DeleteRuleOperation(Rule rule, AttributeValueStoreFactory attributeValueStoreFactory) {
+            super(rule, attributeValueStoreFactory);
+        }
+
+        @Override
+        protected void processAttributeEntry(Map.Entry<Attribute, Set<String>> attributeEntry) {
+            AttributeValueStore<String, String> valueStore = getAttributeValueStore(attributeEntry.getKey());
+            for (String value : attributeEntry.getValue()) {
+                valueStore.remove(value);
+            }
+        }
+    }
+
+    private static class AddRuleOperation extends RuleProcessingOperation {
+        public AddRuleOperation(Rule rule, AttributeValueStoreFactory attributeValueStoreFactory) {
+            super(rule, attributeValueStoreFactory);
+        }
+
+        @Override
+        protected void processAttributeEntry(Map.Entry<Attribute, Set<String>> attributeEntry) {
+            AttributeValueStore<String, String> valueStore = getAttributeValueStore(attributeEntry.getKey());
+            for (String value : attributeEntry.getValue()) {
+                valueStore.put(value, this.rule.getFeatureValue());
+            }
+        }
+    }
+}

libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/AttributeExtractor.java

@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rule.attribute_extractor;
+
+import org.opensearch.autotagging.Attribute;
+
+/**
+ * This interface defines the contract for extracting the attributes for Rule based auto-tagging feature
+ * @param <V>
+ */
+public interface AttributeExtractor<V> {
+    /**
+     * This method returns the Attribute which it is responsible for extracting
+     * @return attribute
+     */
+    Attribute getAttribute();
+
+    /**
+     * This method returns the attribute values in context of the current request
+     * @return attribute value
+     */
+    Iterable<V> extract();
+}

libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package contains feature attribute extractor interface and its implementations
+ */
+package org.opensearch.rule.attribute_extractor;

libs/autotagging-commons/src/main/java/org/opensearch/rule/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Rule based auto-tagging generic constructs
+ */
+package org.opensearch.rule;

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStore.java → libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStore.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.plugin.wlm.rule.storage;
+package org.opensearch.rule.storage;
 
 import java.util.Optional;
 

libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStoreFactory.java

@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.rule.storage;
+
+import org.opensearch.autotagging.Attribute;
+import org.opensearch.autotagging.FeatureType;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * Factory class for AttributeValueStore per feature type as two feature types can potentially share same attribute
+ */
+public class AttributeValueStoreFactory {
+    private final Map<String, AttributeValueStore<String, String>> attributeValueStores = new HashMap<>();
+
+    /**
+     * Constructor
+     * @param featureType  is the feature which are using rule based auto tagging
+     * @param attributeValueStoreSupplier supplies the feature level AttributeValueStore instance
+     */
+    public AttributeValueStoreFactory(FeatureType featureType, Supplier<AttributeValueStore<String, String>> attributeValueStoreSupplier) {
+        for (Attribute attribute : featureType.getAllowedAttributesRegistry().values()) {
+            attributeValueStores.put(attribute.getName(), attributeValueStoreSupplier.get());
+        }
+    }
+
+    /**
+     * Factory method which returns the {@link AttributeValueStore} for the given attribute
+     * @param attribute
+     * @return
+     */
+    public AttributeValueStore<String, String> getAttributeValueStore(final Attribute attribute) {
+        final String attributeName = attribute.getName();
+        if (attributeValueStores == null) {
+            throw new IllegalStateException("AttributeValueStoreFactory is not initialized yet.");
+        }
+
+        if (!attributeValueStores.containsKey(attributeName)) {
+            throw new IllegalArgumentException("[" + attributeName + "] is not a valid attribute for enabled features.");
+        }
+
+        return attributeValueStores.get(attributeName);
+    }
+}

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/DefaultAttributeValueStore.java → libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/DefaultAttributeValueStore.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.plugin.wlm.rule.storage;
+package org.opensearch.rule.storage;
 
 import org.apache.commons.collections4.trie.PatriciaTrie;
 

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/package-info.java → libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/package-info.java

@@ -9,4 +9,4 @@
 /**
  * This package contains interfaces and implementations for in memory rule storage mechanisms
  */
-package org.opensearch.plugin.wlm.rule.storage;
+package org.opensearch.rule.storage;