HDDS-4270. Add more reusable byteman scripts to debug ofs/o3fs performance

dev-support/byteman/appendlog.btm

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#Measure time between GrpcLogAppender.appendLog calls (HB frequency)
+
+RULE GrpcLogAppender.appendLog
+CLASS GrpcLogAppender
+METHOD appendLog
+AT ENTRY
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("appendLog" + Thread.currentThread().getId()))
+IF TRUE
+DO 
+   System.out.println("appendLog: " + elapsedTime);
+   resetTimer("appendLog" + Thread.currentThread().getId());
+ENDRULE

dev-support/byteman/hcfs-read.btm

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure time spent in HCFS read implementations
+
+RULE hcfs-read-FileSystem.close
+CLASS org.apache.hadoop.fs.FileSystem
+METHOD close
+IF TRUE
+DO
+  System.out.println("Closing file system instance: " + System.identityHashCode($0));
+  System.out.println("   read.call: " + readCounter("read.call"));
+  System.out.println("   read.allTime: " + readCounter("read.allTime"));
+  System.out.println("   readFully.call: " + readCounter("readFully.call"));
+  System.out.println("   readFully.allTime: " + readCounter("readFully.allTime"));  
+ENDRULE
+
+
+RULE FSDataInputStream.Read.Entry
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT ENTRY
+IF TRUE
+DO resetTimer("read" + Thread.currentThread().getId());
+   incrementCounter("read.call")
+ENDRULE
+
+RULE FSDataInputStream.Read.Exit
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("read" + Thread.currentThread().getId()))
+IF TRUE
+DO
+   incrementCounter("read.allTime", elapsedTime)
+ENDRULE
+
+RULE FSDataInputStream.ReadFully.Entry
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT ENTRY
+IF TRUE
+DO resetTimer("readFully" + Thread.currentThread().getId());
+   incrementCounter("readFully.call")
+ENDRULE
+
+RULE FSDataInputStream.ReadFully.Exit
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("readFully" + Thread.currentThread().getId()))
+IF TRUE
+DO
+   incrementCounter("readFully.allTime", elapsedTime)
+ENDRULE

dev-support/byteman/ratis-flush.btm

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Print out time spent in Ratis log flush
+
+RULE SegmentedRaftLogOutputStream.flush.Entry
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT ENTRY
+IF TRUE
+DO resetTimer("flush" + Thread.currentThread().getId());
+
+ENDRULE
+
+RULE SegmentedRaftLogOutputStream.flush.Exit
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("flush" + Thread.currentThread().getId()))
+DO
+   System.out.println("flush: " + elapsedTime);
+ENDRULE

dev-support/byteman/ratis-no-flush.btm

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This btm script turns off the flush when Ratis Log is written
+
+RULE Ratis.noFlush
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT ENTRY
+IF TRUE
+DO 
+   return
+ENDRULE

dev-support/byteman/watchforcommit.btm

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure time spent in Watch for commit calls
+
+RULE BlockOutputStream.watchForCommit.Entry
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT ENTRY
+IF TRUE
+DO resetTimer("watchForCommit" + Thread.currentThread().getId());
+
+ENDRULE
+
+RULE BlockOutputStream.watchForCommit.Exit
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("watchForCommit" + Thread.currentThread().getId()))
+IF elapsedTime > 5
+DO 
+   System.out.println("WatchForCommit: " + elapsedTime);
+ENDRULE

dev-support/byteman/watchforcommit_all.btm

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure overall time spent in Watch for commit calls
+
+RULE watchforcommit_all-FileSystem.close
+CLASS org.apache.hadoop.fs.FileSystem
+METHOD close
+IF TRUE
+DO
+  System.out.println("Closing file system instance: " + System.identityHashCode($0));
+  System.out.println("   watchForCommit.call: " + readCounter("watchForCommit.call"));
+  System.out.println("   watchForCommit.allTime: " + readCounter("watchForCommit.allTime"))
+
+ENDRULE
+
+RULE watchforcommit_all-BlockOutputStream.watchForCommit.Entry
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT ENTRY
+IF TRUE
+DO resetTimer(Thread.currentThread());
+   incrementCounter("watchForCommit.call")
+ENDRULE
+
+RULE watchforcommit_all-BlockOutputStream.watchForCommit.Exit
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer(Thread.currentThread()))
+IF TRUE
+DO
+   System.out.println("watchForCommit: " + elapsedTime);
+   incrementCounter("watchForCommit.allTime", elapsedTime)
+ENDRULE