From 1cdeeac79aca034964ce1093d219f3947f5e1e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 23 Sep 2020 14:02:58 +0200 Subject: [PATCH 1/4] HDDS-4270. Add more reusable byteman scripts to debug ofs/o3fs performance --- dev-support/byteman/appendlog.btm | 28 +++++++++ dev-support/byteman/hcfs-read.btm | 67 ++++++++++++++++++++++ dev-support/byteman/ratis-flush.btm | 35 +++++++++++ dev-support/byteman/ratis-no-flush.btm | 25 ++++++++ dev-support/byteman/watchforcommit.btm | 35 +++++++++++ dev-support/byteman/watchforcommit_all.btm | 47 +++++++++++++++ 6 files changed, 237 insertions(+) create mode 100644 dev-support/byteman/appendlog.btm create mode 100644 dev-support/byteman/hcfs-read.btm create mode 100644 dev-support/byteman/ratis-flush.btm create mode 100644 dev-support/byteman/ratis-no-flush.btm create mode 100644 dev-support/byteman/watchforcommit.btm create mode 100644 dev-support/byteman/watchforcommit_all.btm diff --git a/dev-support/byteman/appendlog.btm b/dev-support/byteman/appendlog.btm new file mode 100644 index 000000000000..0744bbcf2691 --- /dev/null +++ b/dev-support/byteman/appendlog.btm @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#Measure time between GrpcLogAppender.appendLog calls (HB frequency) + +RULE GrpcLogAppender.appendLog +CLASS GrpcLogAppender +METHOD appendLog +AT ENTRY +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("appendLog" + Thread.currentThread().getId())) +IF TRUE +DO + System.out.println("appendLog: " + elapsedTime); + resetTimer("appendLog" + Thread.currentThread().getId()); +ENDRULE \ No newline at end of file diff --git a/dev-support/byteman/hcfs-read.btm b/dev-support/byteman/hcfs-read.btm new file mode 100644 index 000000000000..84df0248835b --- /dev/null +++ b/dev-support/byteman/hcfs-read.btm @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Measure time spent in HCFS read implementations + +RULE FileSystem.close +CLASS org.apache.hadoop.fs.FileSystem +METHOD close +IF TRUE +DO + System.out.println("Closing file system instance: " + System.identityHashCode($0)); + System.out.println(" read.call: " + readCounter("read.call")); + System.out.println(" read.allTime: " + readCounter("read.allTime")); + System.out.println(" readFully.call: " + readCounter("readFully.call")); + System.out.println(" readFully.allTime: " + readCounter("readFully.allTime")); +ENDRULE + + +RULE FSDataInputStream.Read.Entry +CLASS org.apache.hadoop.fs.FSDataInputStream +METHOD read +AT ENTRY +IF TRUE +DO resetTimer("read" + Thread.currentThread().getId()); + incrementCounter("read.call") +ENDRULE + +RULE FSDataInputStream.Read.Exit +CLASS org.apache.hadoop.fs.FSDataInputStream +METHOD read +AT EXIT +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("read" + Thread.currentThread().getId())) +IF TRUE +DO + incrementCounter("read.allTime", elapsedTime) +ENDRULE + +RULE FSDataInputStream.ReadFully.Entry +CLASS org.apache.hadoop.fs.FSDataInputStream +METHOD read +AT ENTRY +IF TRUE +DO resetTimer("readFully" + Thread.currentThread().getId()); + incrementCounter("readFully.call") +ENDRULE + +RULE FSDataInputStream.ReadFully.Exit +CLASS org.apache.hadoop.fs.FSDataInputStream +METHOD read +AT EXIT +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("readFully" + Thread.currentThread().getId())) +IF TRUE +DO + incrementCounter("readFully.allTime", elapsedTime) +ENDRULE diff --git a/dev-support/byteman/ratis-flush.btm b/dev-support/byteman/ratis-flush.btm new file mode 100644 index 000000000000..6d340dce9b20 --- /dev/null +++ b/dev-support/byteman/ratis-flush.btm @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Turn of flush for Raits log + + +RULE SegmentedRaftLogOutputStream.flush.Entry +CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream +METHOD flush +AT ENTRY +IF TRUE +DO resetTimer("flush" + Thread.currentThread().getId()); + +ENDRULE + +RULE SegmentedRaftLogOutputStream.flush.Exit +CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream +METHOD flush +AT EXIT +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("flush" + Thread.currentThread().getId())) +DO + System.out.println("flush: " + elapsedTime); +ENDRULE \ No newline at end of file diff --git a/dev-support/byteman/ratis-no-flush.btm b/dev-support/byteman/ratis-no-flush.btm new file mode 100644 index 000000000000..d9d26306819c --- /dev/null +++ b/dev-support/byteman/ratis-no-flush.btm @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This btm script turns off the flush when ratis Log is written + +RULE Ratis.noFlush +CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream +METHOD flush +AT ENTRY +IF TRUE +DO + return +ENDRULE \ No newline at end of file diff --git a/dev-support/byteman/watchforcommit.btm b/dev-support/byteman/watchforcommit.btm new file mode 100644 index 000000000000..9b2e70f87caf --- /dev/null +++ b/dev-support/byteman/watchforcommit.btm @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Measure time spent in Watch for commit calls + +RULE BlockOutputStream.watchForCommit.Entry +CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream +METHOD watchForCommit +AT ENTRY +IF TRUE +DO resetTimer("watchForCommit" + Thread.currentThread().getId()); + +ENDRULE + +RULE BlockOutputStream.watchForCommit.Exit +CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream +METHOD watchForCommit +AT EXIT +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("watchForCommit" + Thread.currentThread().getId())) +IF elapsedTime > 5 +DO + System.out.println("WatchForCommit: " + elapsedTime); +ENDRULE \ No newline at end of file diff --git a/dev-support/byteman/watchforcommit_all.btm b/dev-support/byteman/watchforcommit_all.btm new file mode 100644 index 000000000000..cdb5115b60e6 --- /dev/null +++ b/dev-support/byteman/watchforcommit_all.btm @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Measure overall time spent in Watch for commit calls + +RULE FileSystem.close +CLASS org.apache.hadoop.fs.FileSystem +METHOD close +IF TRUE +DO + System.out.println("Closing file system instance: " + System.identityHashCode($0)); + System.out.println(" watchForCommit.call: " + readCounter("watchForCommit.call")); + System.out.println(" watchForCommit.allTime: " + readCounter("watchForCommit.allTime")) + +ENDRULE + +RULE BlockOutputStream.watchForCommit.Entry +CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream +METHOD watchForCommit +AT ENTRY +IF TRUE +DO resetTimer(Thread.currentThread()); + incrementCounter("watchForCommit.call") +ENDRULE + +RULE BlockOutputStream.watchForCommit.Exit +CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream +METHOD watchForCommit +AT EXIT +BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer(Thread.currentThread())) +IF TRUE +DO + System.out.println("watchForCommit: " + elapsedTime); + incrementCounter("watchForCommit.allTime", elapsedTime) +ENDRULE \ No newline at end of file From cee5eee13396a36f770478d60107fc9d68d4400d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= Date: Thu, 24 Sep 2020 13:16:34 +0200 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- dev-support/byteman/hcfs-read.btm | 2 +- dev-support/byteman/watchforcommit_all.btm | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-support/byteman/hcfs-read.btm b/dev-support/byteman/hcfs-read.btm index 84df0248835b..7200437b2012 100644 --- a/dev-support/byteman/hcfs-read.btm +++ b/dev-support/byteman/hcfs-read.btm @@ -15,7 +15,7 @@ # Measure time spent in HCFS read implementations -RULE FileSystem.close +RULE hcfs-read-FileSystem.close CLASS org.apache.hadoop.fs.FileSystem METHOD close IF TRUE diff --git a/dev-support/byteman/watchforcommit_all.btm b/dev-support/byteman/watchforcommit_all.btm index cdb5115b60e6..85b4d1dcd053 100644 --- a/dev-support/byteman/watchforcommit_all.btm +++ b/dev-support/byteman/watchforcommit_all.btm @@ -15,7 +15,7 @@ # Measure overall time spent in Watch for commit calls -RULE FileSystem.close +RULE watchforcommit_all-FileSystem.close CLASS org.apache.hadoop.fs.FileSystem METHOD close IF TRUE @@ -44,4 +44,4 @@ IF TRUE DO System.out.println("watchForCommit: " + elapsedTime); incrementCounter("watchForCommit.allTime", elapsedTime) -ENDRULE \ No newline at end of file +ENDRULE From 2eb0ef300750bfb8a38cca8d1c4216f6f22a5e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Thu, 24 Sep 2020 13:29:10 +0200 Subject: [PATCH 3/4] Fix docs --- dev-support/byteman/ratis-flush.btm | 7 +++---- dev-support/byteman/ratis-no-flush.btm | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev-support/byteman/ratis-flush.btm b/dev-support/byteman/ratis-flush.btm index 6d340dce9b20..24102a76b3e9 100644 --- a/dev-support/byteman/ratis-flush.btm +++ b/dev-support/byteman/ratis-flush.btm @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Turn of flush for Raits log - +# Print out time spent in Ratis log flush RULE SegmentedRaftLogOutputStream.flush.Entry CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream @@ -30,6 +29,6 @@ CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream METHOD flush AT EXIT BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("flush" + Thread.currentThread().getId())) -DO +DO System.out.println("flush: " + elapsedTime); -ENDRULE \ No newline at end of file +ENDRULE diff --git a/dev-support/byteman/ratis-no-flush.btm b/dev-support/byteman/ratis-no-flush.btm index d9d26306819c..71416d56d812 100644 --- a/dev-support/byteman/ratis-no-flush.btm +++ b/dev-support/byteman/ratis-no-flush.btm @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This btm script turns off the flush when ratis Log is written +# This btm script turns off the flush when Ratis Log is written RULE Ratis.noFlush CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream @@ -22,4 +22,4 @@ AT ENTRY IF TRUE DO return -ENDRULE \ No newline at end of file +ENDRULE From 9897ecb4696b362a55fbacabc47d8ab2021b7dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Thu, 24 Sep 2020 13:31:21 +0200 Subject: [PATCH 4/4] prefix BOS.watchForCommit --- dev-support/byteman/watchforcommit_all.btm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-support/byteman/watchforcommit_all.btm b/dev-support/byteman/watchforcommit_all.btm index 85b4d1dcd053..8e14af8c78d4 100644 --- a/dev-support/byteman/watchforcommit_all.btm +++ b/dev-support/byteman/watchforcommit_all.btm @@ -26,7 +26,7 @@ DO ENDRULE -RULE BlockOutputStream.watchForCommit.Entry +RULE watchforcommit_all-BlockOutputStream.watchForCommit.Entry CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream METHOD watchForCommit AT ENTRY @@ -35,7 +35,7 @@ DO resetTimer(Thread.currentThread()); incrementCounter("watchForCommit.call") ENDRULE -RULE BlockOutputStream.watchForCommit.Exit +RULE watchforcommit_all-BlockOutputStream.watchForCommit.Exit CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream METHOD watchForCommit AT EXIT