OAK-11525 Update fullGC configuration for audit logging into Mongo collection

* OAK-11525 Update fullGC configuration for audit logging into Mongo collection

Author: daniancu

oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java

@@ -134,6 +134,7 @@ private static class RevisionsOptions extends Utils.NodeStoreOptions {
         final OptionSpec<Boolean> dryRun;
         final OptionSpec<Boolean> embeddedVerification;
         final OptionSpec<Integer> fullGcMode;
+        final OptionSpec<Boolean> fullGCAuditLoggingEnabled;
 
         RevisionsOptions(String usage) {
             super(usage);
@@ -195,6 +196,8 @@ private static class RevisionsOptions extends Utils.NodeStoreOptions {
                             "to be considered for Full GC i.e. Version Garbage Collector (Full GC) logic will only consider those " +
                             "nodes for Full GC which are not accessed recently (currentTime - lastModifiedTime > fullGcMaxAge). Default: 86400 (one day)")
                     .withOptionalArg().ofType(Long.class).defaultsTo(TimeUnit.DAYS.toSeconds(1));
+            fullGCAuditLoggingEnabled = parser.accepts("fullGCAuditLoggingEnabled", "Enable audit logging for Full GC")
+                    .withOptionalArg().ofType(Boolean.class).defaultsTo(FALSE);
         }
 
         public RevisionsOptions parse(String[] args) {
@@ -293,6 +296,10 @@ boolean isEntireRepo() {
         boolean doCompaction() {
             return options.has(compact);
         }
+
+        Boolean isFullGCAuditLoggingEnabled() {
+            return options.has(fullGCAuditLoggingEnabled);
+        }
     }
 
     @Override
@@ -358,6 +365,7 @@ private VersionGarbageCollector bootstrapVGC(RevisionsOptions options, Closer cl
         builder.setFullGCBatchSize(options.getFullGcBatchSize());
         builder.setFullGCProgressSize(options.getFullGcProgressSize());
         builder.setFullGcMaxAgeMillis(SECONDS.toMillis(options.getFullGcMaxAge()));
+        builder.setFullGCAuditLoggingEnabled(options.isFullGCAuditLoggingEnabled());
 
         // create a VersionGCSupport while builder is read-write
         VersionGCSupport gcSupport = builder.createVersionGCSupport();
@@ -391,6 +399,7 @@ private VersionGarbageCollector bootstrapVGC(RevisionsOptions options, Closer cl
         System.out.println("FullGcProgressSize is : " + options.getFullGcProgressSize());
         System.out.println("FullGcMaxAgeInSecs is : " + options.getFullGcMaxAge());
         System.out.println("FullGcMaxAgeMillis is : " + builder.getFullGcMaxAgeMillis());
+        System.out.println("FullGCAuditLoggingEnabled is : " + options.isFullGCAuditLoggingEnabled());
         VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, options.isDryRun(), builder);
 
         VersionGCOptions gcOptions = gc.getOptions();

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java

@@ -400,4 +400,9 @@
             name = "Invisible for discovery",
             description = "Boolean value indicating whether the instance should be discoverable by the cluster. The default value is " + DocumentNodeStoreService.DEFAULT_INVISIBLE_FOR_DISCOVERY)
     boolean invisibleForDiscovery() default DocumentNodeStoreService.DEFAULT_INVISIBLE_FOR_DISCOVERY;
+
+    @AttributeDefinition(
+        name = "Enable Full GC Persistent Audit Logging",
+        description = "This parameter will enable/disable the saving of deleted document IDs and properties during FullGC into a persistent storage, e.g Mongo collection")
+    boolean fullGCAuditLoggingEnabled() default false;
 }

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java

@@ -185,6 +185,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
     private int fullGCBatchSize = DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
     private int fullGCProgressSize = DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
     private double fullGCDelayFactor = DocumentNodeStoreService.DEFAULT_FGC_DELAY_FACTOR;
+    private boolean fullGCAuditLoggingEnabled;
     private long suspendTimeoutMillis = DEFAULT_SUSPEND_TIMEOUT;
 
     /**
@@ -317,6 +318,15 @@ public boolean isFullGCEnabled() {
         return this.fullGCEnabled;
     }
 
+    public T setFullGCAuditLoggingEnabled(boolean b) {
+        this.fullGCAuditLoggingEnabled = b;
+        return thisBuilder();
+    }
+
+    public boolean isFullGCAuditLoggingEnabled() {
+        return this.fullGCAuditLoggingEnabled;
+    }
+
     public T setFullGCIncludePaths(@Nullable String[] includePaths) {
         if (isNull(includePaths) || includePaths.length == 0 || Arrays.equals(includePaths, new String[]{"/"})) {
             this.fullGCIncludePaths = Set.of();

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java

@@ -534,6 +534,7 @@ private void configureBuilder(DocumentNodeStoreBuilder<?> builder) {
                 setFullGCBatchSize(config.fullGCBatchSize()).
                 setFullGCProgressSize(config.fullGCProgressSize()).
                 setFullGCDelayFactor(config.fullGCDelayFactor()).
+                setFullGCAuditLoggingEnabled(config.fullGCAuditLoggingEnabled()).
                 setSuspendTimeoutMillis(config.suspendTimeoutMillis()).
                 setClusterIdReuseDelayAfterRecovery(config.clusterIdReuseDelayAfterRecoveryMillis()).
                 setRecoveryDelayMillis(config.recoveryDelayMillis()).

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderBase.java

@@ -172,7 +172,7 @@ public T setCollectionCompressionType(String compressionType) {
     public VersionGCSupport createVersionGCSupport() {
         DocumentStore store = getDocumentStore();
         if (store instanceof MongoDocumentStore) {
-            return new MongoVersionGCSupport((MongoDocumentStore) store);
+            return new MongoVersionGCSupport((MongoDocumentStore) store, isFullGCAuditLoggingEnabled());
         } else {
             return super.createVersionGCSupport();
         }

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java

@@ -351,6 +351,9 @@ public MongoDocumentStore(MongoClient connection, MongoDatabase db,
 
         if (!readOnly) {
             ensureIndexes(db, status);
+            if (builder.isFullGCAuditLoggingEnabled()) {
+                ensureFullGcTTLIndex();
+            }
         }
 
         this.nodeLocks = new StripedNodeDocumentLocks();
@@ -466,10 +469,12 @@ private void ensureIndexes(@NotNull MongoDatabase db, @NotNull MongoStatus mongo
 
         // index on _modified for journal entries
         createIndex(journal, JournalEntry.MODIFIED, true, false, false);
+    }
 
+    private void ensureFullGcTTLIndex() {
         //TTL index for full GC bin documents to expire after 90 days
         //see https://issues.apache.org/jira/browse/OAK-11444
-        IndexOptions indexOptions = new IndexOptions().expireAfter(TimeUnit.DAYS.toSeconds(90), java.util.concurrent.TimeUnit.SECONDS);
+        IndexOptions indexOptions = new IndexOptions().expireAfter(TimeUnit.DAYS.toSeconds(90), TimeUnit.SECONDS);
         connection.getCollection(BIN_COLLECTION).createIndex(new org.bson.Document(MongoFullGcNodeBin.GC_COLLECTED_AT, 1), indexOptions);
     }
 

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoFullGcNodeBin.java

@@ -17,7 +17,6 @@
 package org.apache.jackrabbit.oak.plugins.document.mongo;
 
 import com.mongodb.BasicDBObject;
-import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
 import org.apache.jackrabbit.oak.plugins.document.Collection;
 import org.apache.jackrabbit.oak.plugins.document.Document;
 import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
@@ -46,13 +45,19 @@
  *  Each method delegates directly to DocumentStore
  */
 public class MongoFullGcNodeBin implements FullGcNodeBin {
-    private static final Logger LOG = LoggerFactory.getLogger(MongoFullGcNodeBin.class);
     public static final String GC_COLLECTED_AT = "_gcCollectedAt";
+    private static final Logger LOG = LoggerFactory.getLogger(MongoFullGcNodeBin.class);
+
     private final MongoDocumentStore mongoDocumentStore;
-    private boolean enabled = SystemPropertySupplier.create("oak.document.fullGcBin.enabled", false).get();
+    private boolean enabled;
 
     public MongoFullGcNodeBin(MongoDocumentStore ds) {
-        mongoDocumentStore = ds;
+        this(ds, false);
+    }
+
+    public MongoFullGcNodeBin(MongoDocumentStore store, boolean fullGcBinEnabled) {
+        mongoDocumentStore = store;
+        enabled = fullGcBinEnabled;
     }
 
     /**
@@ -100,15 +105,9 @@ public List<NodeDocument> findAndUpdate(List<UpdateOp> updateOpList) {
         return mongoDocumentStore.findAndUpdate(Collection.NODES, updateOpList);
     }
 
-    /**
-     * Saves the name of properties that will be removed  in the BIN collection
-     *
-     * @param orphanOrDeletedRemovalMap the keys of the documents to remove with the corresponding timestamps
-     * @return true if the documents were successfully added to the bin
-     */
     private boolean addToBin(Map<String, Long> orphanOrDeletedRemovalMap) {
         if (!enabled) {
-            LOG.info("Bin is disabled, skip adding delete candidate documents to bin");
+            LOG.info("Bin is disabled, skipping adding delete candidate documents to bin");
             return true;
         }
         LOG.info("Adding {} delete candidate documents to bin", orphanOrDeletedRemovalMap.size());
@@ -124,15 +123,9 @@ private boolean addToBin(Map<String, Long> orphanOrDeletedRemovalMap) {
         return false;
     }
 
-    /**
-     * Saves the ID of documents that will be removed in the BIN collection
-     *
-     * @param updateOpList the update operation list for removing the documents
-     * @return true if the documents were successfully added to the bin
-     */
     private boolean addToBin(List<UpdateOp> updateOpList) {
         if (!enabled) {
-            LOG.info("Bin is disabled, skip adding removed properties to bin");
+            LOG.info("Bin is disabled, skipping adding removed properties to bin");
             return true;
         }
         LOG.info("Adding {} removed properties to bin", updateOpList.size());
@@ -169,5 +162,10 @@ private BasicDBObject toBasicDBObject(UpdateOp op) {
     @Override
     public void setEnabled(boolean value) {
         this.enabled = value;
+        LOG.info("Full GC Bin changed to {}", enabled ? "enabled" : "disabled");
+    }
+
+    public boolean isEnabled() {
+        return enabled;
     }
 }

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java

@@ -29,6 +29,7 @@
 import static com.mongodb.client.model.Filters.and;
 import static com.mongodb.client.model.Filters.lt;
 import static java.util.Collections.emptyList;
+import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
 import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
 import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
 import org.apache.jackrabbit.oak.plugins.document.FullGcNodeBin;
@@ -100,7 +101,6 @@ public class MongoVersionGCSupport extends VersionGCSupport {
 
     /** the hint representing "_modified_1__id_1" - if that index exists, null otherwise */
     private final BasicDBObject modifiedIdHint;
-    private final MongoFullGcNodeBin fullGcNodeBin;
 
     /** timestamp of last time an explain of the 'getModifiedDocs' query was logged */
     private long lastExplainLogMs = -1;
@@ -111,10 +111,15 @@ public class MongoVersionGCSupport extends VersionGCSupport {
     /**
      * The batch size for the query of possibly deleted docs.
      */
-    private final int batchSize = Integer.getInteger(
-            "oak.mongo.queryDeletedDocsBatchSize", 1000);
+    private final int batchSize = SystemPropertySupplier.create(
+        "oak.mongo.queryDeletedDocsBatchSize", 1000).get();
+    private final MongoFullGcNodeBin fullGcBin;
 
     public MongoVersionGCSupport(MongoDocumentStore store) {
+        this(store, false);
+    }
+
+    public MongoVersionGCSupport(MongoDocumentStore store, boolean fullGcBinEnabled) {
         super(store);
         this.store = store;
         if(hasIndex(getNodeCollection(), SD_TYPE, SD_MAX_REV_TIME_IN_SECS)) {
@@ -131,7 +136,7 @@ public MongoVersionGCSupport(MongoDocumentStore store) {
         } else {
             modifiedIdHint = null;
         }
-        fullGcNodeBin = new MongoFullGcNodeBin(store);
+        this.fullGcBin = new MongoFullGcNodeBin(store, fullGcBinEnabled);
     }
 
     @Override
@@ -485,7 +490,7 @@ private void logSplitDocIdsTobeDeleted(Bson query) {
 
     @Override
     public FullGcNodeBin getFullGCBin() {
-        return fullGcNodeBin;
+        return fullGcBin;
     }
 
     private static String getID(BasicDBObject document) {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java

@@ -136,6 +136,18 @@ public RDBDocumentNodeStoreBuilder setFullGCEnabled(boolean b) {
         return thisBuilder();
     }
 
+    @Override
+    public boolean isFullGCAuditLoggingEnabled() {
+        // fullGC is non supported for RDB
+        return false;
+    }
+
+    @Override
+    public RDBDocumentNodeStoreBuilder setFullGCAuditLoggingEnabled(boolean b) {
+        // fullGC is non supported for RDB
+        return thisBuilder();
+    }
+
     @Override
     public Set<String> getFullGCIncludePaths() {
         return of();

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java

@@ -104,6 +104,7 @@ public void defaultValues() throws Exception {
         assertEquals(DEFAULT_EMBEDDED_VERIFICATION_ENABLED, config.embeddedVerificationEnabled());
         assertEquals(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE, config.fullGcMaxAgeInSecs());
         assertEquals(CommitQueue.DEFAULT_SUSPEND_TIMEOUT, config.suspendTimeoutMillis());
+        assertFalse(config.fullGCAuditLoggingEnabled());
     }
 
     @Test
@@ -170,6 +171,13 @@ public void fullGCBatchSize() throws Exception {
         assertEquals(batchSize, config.fullGCBatchSize());
     }
 
+    @Test
+    public void fullGCAuditLoggingEnabled() throws Exception {
+        addConfigurationEntry(preset, "fullGCAuditLoggingEnabled", true);
+        Configuration config = createConfiguration();
+        assertTrue(config.fullGCAuditLoggingEnabled());
+    }
+
     @Test
     public void invisibleForDiscoveryFalse() throws Exception {
         boolean batchSize = false;

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java

@@ -132,4 +132,10 @@ public void fullGCModeDefaultValue() {
         final int fullGcModeNone = 0;
         assertEquals(builder.getFullGCMode(), fullGcModeNone);
     }
+
+    @Test
+    public void isFullGCAuditLoggingEnabled() {
+        MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
+        assertFalse(builder.isFullGCAuditLoggingEnabled());
+    }
 }

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoFullGcNodeBinTest.java

@@ -21,8 +21,10 @@
 import com.mongodb.client.MongoCollection;
 import org.apache.jackrabbit.oak.plugins.document.Collection;
 import org.apache.jackrabbit.oak.plugins.document.Document;
+import org.apache.jackrabbit.oak.plugins.document.FullGcNodeBin;
 import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
 import org.apache.jackrabbit.oak.plugins.document.UpdateOp;
+import org.junit.After;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -53,7 +55,7 @@ public class MongoFullGcNodeBinTest {
     @Mock
     MongoDocumentStore documentStore;
 
-    @InjectMocks
+
     MongoFullGcNodeBin fullGcBin;
 
     @Mock MongoCollection<BasicDBObject> mockBinCollection;
@@ -62,6 +64,7 @@ public class MongoFullGcNodeBinTest {
     @Before
     public void setUp() throws Exception {
         MockitoAnnotations.openMocks(this);
+        fullGcBin = new MongoFullGcNodeBin(documentStore, true);
         when(documentStore.remove(eq(Collection.NODES), anyMap())).thenAnswer(invocation -> {
             Map<String, Long> map = invocation.getArgument(1);
             return map.size();
@@ -72,8 +75,21 @@ public void setUp() throws Exception {
         });
 
         when(documentStore.getBinCollection()).thenReturn(mockBinCollection);
+    }
+
+    @After
+    public void tearDown() {
+        Mockito.reset(documentStore, mockBinCollection);
+    }
 
-        fullGcBin.setEnabled(true);
+    @Test
+    public void defaultDisabled() {
+        assertFalse(new MongoFullGcNodeBin(this.documentStore).isEnabled());
+    }
+
+    @Test
+    public void enableWithConstructor() {
+        assertTrue(new MongoFullGcNodeBin(this.documentStore, true).isEnabled());
     }
 
     @Test

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java

@@ -121,4 +121,11 @@ public void fullGcMaxAgeInSecsHasDefaultValue() {
         builder.setFullGcMaxAgeMillis(30 * 24 * 60 * 60 * 1000L);
         assertEquals(0, builder.getFullGcMaxAgeMillis());
     }
+
+    @Test
+    public void fullGcAuditLoggingEnabled() {
+        RDBDocumentNodeStoreBuilder builder = new RDBDocumentNodeStoreBuilder();
+        builder.setFullGCAuditLoggingEnabled(true);
+        assertFalse(builder.isFullGCAuditLoggingEnabled());
+    }
 }