HDDS-13251. Support dynamic Byteman scripts via bmsubmit in ozonesecure-ha

dev-support/byteman/skip-notify-group-remove.btm

@@ -0,0 +1,10 @@
+RULE skip notifyGroupRemove
+CLASS org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine
+METHOD notifyGroupRemove
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Skip notifyGroupRemove in ContainerStateMachine");
+    return;
+ENDRULE

dev-support/byteman/skip-put-block.btm

@@ -0,0 +1,10 @@
+RULE Block putBlock
+CLASS org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl
+METHOD putBlock
+AT ENTRY
+IF TRUE
+DO
+    System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+    "Blocking putBlock in BlockManagerImpl");
+    return 0;
+ENDRULE

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env

@@ -27,3 +27,5 @@ RANGER_DB_IMAGE_VERSION=12
 RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
 RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
 RANGER_VERSION=${ranger.version}
+BYTEMAN_PORT=9091
+BYTEMAN_HOME=/opt/byteman/

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/byteman.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+x-byteman-config:
+  &byteman-config
+  environment:
+    BYTEMAN_HOME: /opt/byteman/
+    OZONE_OPTS: -javaagent:/opt/byteman.jar=listener:true,interface:0.0.0.0,port:${BYTEMAN_PORT}
+    BYTEMAN_PORT: ${BYTEMAN_PORT}
+
+services:
+  datanode1:
+    <<: *byteman-config
+  datanode2:
+    <<: *byteman-config
+  datanode3:
+    <<: *byteman-config
+  om1:
+    <<: *byteman-config
+  om2:
+    <<: *byteman-config
+  om3:
+    <<: *byteman-config
+  httpfs:
+    <<: *byteman-config
+  s3g:
+    <<: *byteman-config
+  scm1.org:
+    <<: *byteman-config
+  scm2.org:
+    <<: *byteman-config
+  scm3.org:
+    <<: *byteman-config
+  recon:
+    <<: *byteman-config

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-byteman.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:misc
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:byteman.yaml
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+## Run virtual host test cases
+execute_robot_test om1 ozone-fi/byteman_faults_sample.robot

hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional consolermation
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import json
+from robot.api import logger
+
+class BytemanLibrary:
+    def __init__(self):
+        self.byteman_clients = {}
+
+    def connect_to_byteman_agent(self, component_name, host="localhost", port=9091):
+        """Connect to Byteman agent on specific component"""
+        self.byteman_clients[component_name] = {'host': host, 'port': port, 
+                                                'base_url': f"http://{host}:{port}"}
+        logger.console(f"Connected to Byteman agent for {component_name} at {host}:{port}")
+
+    def add_byteman_rule(self, component_name, rule_file):
+        """Add Byteman rule into specific component"""
+        client = self.byteman_clients[component_name]
+
+        # Use bmsubmit command to load rule
+        cmd = ["bmsubmit", "-h", component_name, "-p", str(client['port']), "-l", rule_file]
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode != 0:
+            raise RuntimeError(f"Failed to load rule: {result.stderr}")
+        logger.console(f"Loaded Byteman rule {rule_file} into {component_name}")
+
+    def remove_byteman_rule(self, component_name, rule_file):
+        """Remove Byteman rule for specific component"""
+        client = self.byteman_clients[component_name]
+
+        cmd = ["bmsubmit", "-h", component_name, "-p", str(client['port']), "-u", rule_file]
+        logger.console(cmd)
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode != 0:
+            raise RuntimeError(f"Failed to unload rule: {result.stderr}")
+        logger.console(f"Unloaded Byteman rule {rule_file} from {component_name}")
+
+    def list_byteman_rules(self, component_name):
+        """List all Byteman rules for specific component"""
+        client = self.byteman_clients[component_name]
+
+        cmd = ["bmsubmit", "-h", component_name, "-p", str(client['port']), "-l"]
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+        logger.console(f"Active rules in {component_name}: {result.stdout}")
+        return result.stdout

hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot

@@ -0,0 +1,52 @@
+*** Settings ***
+Library    ../lib/BytemanLibrary.py
+Library    String
+
+*** Variables ***
+${BYTEMAN_PORT}                   9091
+${DATANODE1_BYTEMAN_HOST_PORT}    datanode1:${BYTEMAN_PORT}
+${DATANODE2_BYTEMAN_HOST_PORT}    datanode2:${BYTEMAN_PORT}
+${DATANODE3_BYTEMAN_HOST_PORT}    datanode3:${BYTEMAN_PORT}
+${OM1_BYTEMAN_HOST_PORT}          om1:${BYTEMAN_PORT}
+${OM2_BYTEMAN_HOST_PORT}          om2:${BYTEMAN_PORT}
+${OM3_BYTEMAN_HOST_PORT}          om3:${BYTEMAN_PORT}
+${RECON_BYTEMAN_HOST_PORT}        recon:${BYTEMAN_PORT}
+${SCM1_BYTEMAN_HOST_PORT}         scm1.org:${BYTEMAN_PORT}
+${SCM2_BYTEMAN_HOST_PORT}         scm2.org:${BYTEMAN_PORT}
+${SCM3_BYTEMAN_HOST_PORT}         scm3:${BYTEMAN_PORT}
+${HTTPFS_BYTEMAN_HOST_PORT}       httpfs:${BYTEMAN_PORT}
+${S3G_BYTEMAN_HOST_PORT}          s3g:${BYTEMAN_PORT}
+
+*** Keywords ***
+Setup Byteman For Component
+    [Arguments]    ${component}    ${host_port}
+    ${host}    ${port} =    Split String    ${host_port}    :
+    Connect To Byteman Agent    ${component}    ${host}    ${port}
+
+Setup All Byteman Agents
+    Log   Inside Setup All Byteman Agents
+    Setup Byteman For Component    datanode1   ${DATANODE1_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    datanode2   ${DATANODE2_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    datanode3   ${DATANODE3_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    om1         ${OM1_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    om2         ${OM2_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    om3         ${OM3_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    recon       ${RECON_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    scm1        ${SCM1_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    scm2        ${SCM2_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    scm3        ${SCM3_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    https       ${HTTPFS_BYTEMAN_HOST_PORT}
+    Setup Byteman For Component    s3g         ${S3G_BYTEMAN_HOST_PORT}
+
+Inject Fault Into Component
+    [Arguments]    ${component}    ${rule_file}
+    Add Byteman Rule    ${component}    ${rule_file}
+
+Remove Fault From Component
+    [Arguments]    ${component}    ${rule_file}
+    Remove Byteman Rule    ${component}    ${rule_file}
+
+Verify Byteman Rules Active
+    [Arguments]    ${component}
+    ${rules} =    List all Byteman Rules    ${component}
+    Should Not Be Empty    ${rules}

hadoop-ozone/dist/src/main/smoketest/ozone-fi/byteman_faults_sample.robot

@@ -0,0 +1,32 @@
+*** Variables ***
+${RULE}    /opt/hadoop/share/ozone/byteman/skip-put-block.btm
+${vol}     vol1
+${buck}    buck1
+${key}     key1
+${SECURITY_ENABLED}  true
+
+*** Settings ***
+Resource            BytemanKeywords.robot
+Resource            os.robot
+Resource            ../lib/os.robot
+Resource            ../commonlib.robot
+Resource            ../ozone-lib/shell.robot
+Suite Setup         Setup Suite
+Suite Teardown      Teardown Suite
+
+
+*** Keywords ***
+Setup Suite
+    Setup All Byteman Agents
+    Inject Fault Into Component    datanode1    ${RULE}
+    Log To Console      Kinit
+    Execute And Ignore Error     kinit -kt /etc/security/keytabs/testuser.keytab testuser/om
+
+
+Teardown Suite
+    Remove Fault From Component    datanode1    ${RULE}
+
+*** Test Cases ***
+Print Byteman Port
+    ${BYTEMAN_PORT} =    Get Environment Variable    BYTEMAN_PORT
+    Log    ${BYTEMAN_PORT}