HDDS-13251. Support dynamic Byteman scripts via bmsubmit in ozonesecure-ha

dev-support/byteman/skip-notify-group-remove.btm

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE skip notifyGroupRemove
+CLASS org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine
+METHOD notifyGroupRemove
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Skip notifyGroupRemove in ContainerStateMachine");
+    return;
+ENDRULE

dev-support/byteman/skip-put-block.btm

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE Block putBlock
+CLASS org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl
+METHOD putBlock
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Blocking putBlock in BlockManagerImpl");
+    return 0;
+ENDRULE

hadoop-ozone/dist/pom.xml

@@ -25,7 +25,7 @@
   <name>Apache Ozone Distribution</name>
   <properties>
     <!-- suffix appended to Ozone version to get Docker image version -->
-    <docker.ozone-runner.version>20241216-1-jdk21</docker.ozone-runner.version>
+    <docker.ozone-runner.version>20250625-1-jdk21</docker.ozone-runner.version>
     <docker.ozone-testkr5b.image>ghcr.io/apache/ozone-testkrb5:20241129-1</docker.ozone-testkr5b.image>
     <docker.ozone.image>apache/ozone</docker.ozone.image>
     <docker.ozone.image.flavor>-rocky</docker.ozone.image.flavor>

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env

@@ -27,3 +27,5 @@ RANGER_DB_IMAGE_VERSION=12
 RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
 RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
 RANGER_VERSION=${ranger.version}
+BYTEMAN_PORT=9091
+BYTEMAN_HOME=/opt/byteman/

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/byteman.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+x-byteman-config:
+  &byteman-config
+  environment:
+    BYTEMAN_HOME: /opt/byteman/
+    OZONE_SERVER_OPTS: -javaagent:/opt/byteman.jar=listener:true,address:0.0.0.0,port:${BYTEMAN_PORT}
+    BYTEMAN_PORT: ${BYTEMAN_PORT}
+
+services:
+  datanode1:
+    <<: *byteman-config
+  datanode2:
+    <<: *byteman-config
+  datanode3:
+    <<: *byteman-config
+  om1:
+    <<: *byteman-config
+  om2:
+    <<: *byteman-config
+  om3:
+    <<: *byteman-config
+  httpfs:
+    <<: *byteman-config
+  s3g:
+    <<: *byteman-config
+  scm1.org:
+    <<: *byteman-config
+  scm2.org:
+    <<: *byteman-config
+  scm3.org:
+    <<: *byteman-config
+  recon:
+    <<: *byteman-config

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-byteman.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:HA-secure
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:byteman.yaml
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+## Run virtual host test cases
+execute_robot_test om1 ozone-fi/byteman_faults_sample.robot

hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import os
+from robot.api import logger
+
+class BytemanLibrary:
+    def __init__(self):
+        self.byteman_port = os.getenv("BYTEMAN_PORT", "9091")
+        self.byteman_cmd = ["bmsubmit", "-p", self.byteman_port]
+
+    def update_component_name(self, component_name):
+        return f"{component_name}.org" if component_name in ["scm1", "scm2", "scm3"] else component_name
+
+    def run_byteman_cmd(self, component_name, args, action_desc):
+        """Run a byteman command and handle error/logging"""
+        cmd = self.byteman_cmd + ["-h", component_name] + args
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            raise RuntimeError(f"Failed to {action_desc} for {component_name}: {result.stderr.strip()}")
+
+        logger.info(f"{action_desc} for {component_name} successful.")
+        return result.stdout
+
+    def add_byteman_rule(self, component_name, rule_file):
+        """Add Byteman rule into specific component"""
+        component_name = self.update_component_name(component_name)
+        self.run_byteman_cmd(component_name, ["-l", rule_file], f"Add rule {rule_file}")
+
+    def remove_byteman_rule(self, component_name, rule_file):
+        """Remove Byteman rule for specific component"""
+        component_name = self.update_component_name(component_name)
+        self.run_byteman_cmd(component_name, ["-u", rule_file], f"Remove rule {rule_file}")
+
+    def list_byteman_rules(self, component_name):
+        """List Active Byteman rules for specific component and return file list"""
+        component_name = self.update_component_name(component_name)
+        output = self.run_byteman_cmd(component_name, ["-l"], "List rules")
+
+        matching_lines = [line for line in output.splitlines() if '# File' in line]
+        file_list = [line.split()[2] for line in matching_lines if len(line.split()) >= 3]
+
+        if matching_lines:
+            logger.info(f"Active rules in {component_name}:\n" + "\n".join(matching_lines))
+        else:
+            logger.info(f"Active rules in {component_name}: No rules found")
+
+        return file_list
+
+    def remove_all_byteman_rules(self, component_name):
+        """Remove all Byteman rules for specific component"""
+        component_name = self.update_component_name(component_name)
+        rule_files = self.list_byteman_rules(component_name)
+
+        if not rule_files:
+            logger.info(f"No active rules to remove for {component_name}")
+            return
+
+        for rule_file in rule_files:
+            self.remove_byteman_rule(component_name, rule_file)

hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot

@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+*** Settings ***
+Library    ../lib/BytemanLibrary.py
+Library    String
+
+
+*** Variables ***
+@{ALL_COMPONENTS}         datanode1    datanode2    datanode3    om1    om2    om3    recon    scm1    scm2    scm3    s3g
+@{DATANODE_COMPONENTS}    datanode1    datanode2    datanode3
+@{OM_COMPONENTS}          om1    om2    om3
+@{SCM_COMPONENTS}         scm1    scm2    scm3
+
+*** Keywords ***
+Inject Fault Into All Components
+    [Arguments]    ${rule_file}
+    Log    Injecting fault ${rule_file} into all components
+    FOR    ${component}    IN    @{ALL_COMPONENTS}
+        Run Keyword And Continue On Failure    Add Byteman Rule    ${component}    ${rule_file}
+    END
+
+
+Remove Fault From All Components
+    [Arguments]    ${rule_file}
+    Log    Removing fault ${rule_file} from all components
+    FOR    ${component}    IN    @{ALL_COMPONENTS}
+        Run Keyword And Continue On Failure    Remove Byteman Rule    ${component}    ${rule_file}
+    END
+
+
+List Byteman Rules for All Components
+    Log    Listing active rules for all components
+    FOR    ${component}    IN    @{ALL_COMPONENTS}
+        Run Keyword And Continue On Failure    List Byteman Rules    ${component}
+    END
+
+
+Remove All Rules From All Components
+    Log    Removing all rules from all components
+    FOR    ${component}    IN    @{ALL_COMPONENTS}
+        Run Keyword And Continue On Failure    Remove All Byteman Rules    ${component}
+    END
+
+
+Inject Fault Into Datanodes Only
+    [Arguments]    ${rule_file}
+    Log    Injecting fault ${rule_file} into datanodes only
+    FOR    ${component}    IN    @{DATANODE_COMPONENTS}
+        Run Keyword And Continue On Failure    Add Byteman Rule    ${component}    ${rule_file}
+    END
+
+List Byteman Rules for Datanodes
+    Log    Listing active rules for all datanodes
+    FOR    ${component}    IN    @{DATANODE_COMPONENTS}
+        Run Keyword And Continue On Failure    List Byteman Rules    ${component}
+    END
+
+Remove Fault From Datanodes Only
+    [Arguments]    ${rule_file}
+    Log    Removing fault ${rule_file} from datanodes only
+    FOR    ${component}    IN    @{DATANODE_COMPONENTS}
+        Run Keyword And Continue On Failure    Remove Byteman Rule    ${component}    ${rule_file}
+    END
+
+
+Inject Fault Into OMs Only
+    [Arguments]    ${rule_file}
+    Log    Injecting fault ${rule_file} into oms only
+    FOR    ${component}    IN    @{OM_COMPONENTS}
+        Run Keyword And Continue On Failure    Add Byteman Rule    ${component}    ${rule_file}
+    END
+
+List Byteman Rules for OMs
+    Log    Listing active rules for all OMs
+    FOR    ${component}    IN    @{OM_COMPONENTS}
+        Run Keyword And Continue On Failure    List Byteman Rules    ${component}
+    END
+
+Remove Fault From OMs Only
+    [Arguments]    ${rule_file}
+    Log    Removing fault ${rule_file} from oms only
+    FOR    ${component}    IN    @{OM_COMPONENTS}
+        Run Keyword And Continue On Failure    Remove Byteman Rule    ${component}    ${rule_file}
+    END
+
+
+Inject Fault Into SCMs Only
+    [Arguments]    ${rule_file}
+    Log    Injecting fault ${rule_file} into scms only
+    FOR    ${component}    IN    @{SCM_COMPONENTS}
+        Run Keyword And Continue On Failure    Add Byteman Rule    ${component}    ${rule_file}
+    END
+
+List Byteman Rules for SCMs
+    Log    Listing active rules for all SCMs
+    FOR    ${component}    IN    @{SCM_COMPONENTS}
+        Run Keyword And Continue On Failure    List Byteman Rules    ${component}
+    END
+
+Remove Fault From SCMs Only
+    [Arguments]    ${rule_file}
+    Log    Removing fault ${rule_file} from scms only
+    FOR    ${component}    IN    @{SCM_COMPONENTS}
+        Run Keyword And Continue On Failure    Remove Byteman Rule    ${component}    ${rule_file}
+    END