From 6054426f70e2fa29dd2af8619bfe8f580d8f0a2a Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Fri, 10 Mar 2023 16:21:53 +0800 Subject: [PATCH 1/2] Add Configuration for OM Ratis Log Purge tuning parameters --- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 9 +++++++++ .../ozone/om/ratis/OzoneManagerRatisServer.java | 16 +++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 9ba5485da276..5716f0623775 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -151,6 +151,15 @@ private OMConfigKeys() { public static final String OZONE_OM_RATIS_LOG_PURGE_GAP = "ozone.om.ratis.log.purge.gap"; public static final int OZONE_OM_RATIS_LOG_PURGE_GAP_DEFAULT = 1000000; + public static final String OZONE_OM_RATIS_LOG_PURGE_UPTO_SNAPSHOT_INDEX + = "ozone.om.ratis.log.purge.upto.snapshot.index"; + public static final boolean + OZONE_OM_RATIS_LOG_PURGE_UPTO_SNAPSHOT_INDEX_DEFAULT = true; + public static final String + OZONE_OM_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM + = "ozone.om.ratis.log.purge.preservation.log.num"; + public static final long + OZONE_OM_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT = 0L; public static final String OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY = "ozone.om.ratis.snapshot.auto.trigger.threshold"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 9ab6ecdad272..aa4c9b1db83c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -602,7 +602,21 @@ private RaftProperties newRaftProperties(ConfigurationSource conf) { StorageUnit.BYTES); RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf(raftSegmentSize)); - RaftServerConfigKeys.Log.setPurgeUptoSnapshotIndex(properties, true); + + // Set to enable RAFT to purge logs up to Snapshot Index + RaftServerConfigKeys.Log.setPurgeUptoSnapshotIndex(properties, + conf.getBoolean( + OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_UPTO_SNAPSHOT_INDEX, + OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_UPTO_SNAPSHOT_INDEX_DEFAULT + ) + ); + // Set number of last RAFT logs to not be purged + RaftServerConfigKeys.Log.setPurgePreservationLogNum(properties, + conf.getLong( + OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM, + OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT + ) + ); // Set RAFT segment pre-allocated size final long raftSegmentPreallocatedSize = (long) conf.getStorageSize( From 84514b3f578baa87027e47d8a01573e85a6d9c3f Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 11 Mar 2023 15:21:33 +0800 Subject: [PATCH 2/2] Add the configurations to ozone-default.xml --- .../src/main/resources/ozone-default.xml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f093bf7303bf..5541d910776d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1864,7 +1864,23 @@ its log segments after taking snapshot. - + + ozone.om.ratis.log.purge.upto.snapshot.index + true + OZONE, OM, RATIS + + Enable/disable Raft server to purge its log up to the snapshot index + after taking snapshot. + + + + ozone.om.ratis.log.purge.preservation.log.num + 0 + OZONE, OM, RATIS + + The number of latest Raft logs to not be purged after taking snapshot. + + ozone.om.ratis.server.request.timeout 3s