Skip to content

HADOOP-18631 (ADDENDUM) Use LogCapturer to match audit log pattern and remove hdfs async audit log configs #5451

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jojochuang merged 9 commits into from
Mar 17, 2023
Merged

HADOOP-18631 (ADDENDUM) Use LogCapturer to match audit log pattern and remove hdfs async audit log configs #5451

Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8a834fe
HADOOP-18631 (ADDENDUM) Avoid deleting audit log file under heavy writes
virajjasani Mar 3, 2023
257f8f6
addendum
virajjasani Mar 3, 2023
167274c
addendum
virajjasani Mar 3, 2023
6740a97
minor change to use before and after class to initiate and stop capture
virajjasani Mar 5, 2023
2018839
addendum
virajjasani Mar 5, 2023
1483940
TestAuditLogs to use LogCapturer
virajjasani Mar 5, 2023
86ae052
remove redundant comparison
virajjasani Mar 6, 2023
af017d1
remove configs
virajjasani Mar 15, 2023
6ecd8d9
addendum
virajjasani Mar 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 26 additions & 32 deletions ...fs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
Expand Down Expand Up @@ -117,9 +115,9 @@
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -132,10 +130,7 @@ public class TestFsck {
private static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(TestFsck.class.getName());

private static final File AUDIT_LOG_FILE =
new File(System.getProperty("hadoop.log.dir"), "hdfs-audit.log");

// Pattern for:
// Pattern for:
// allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null
static final Pattern FSCK_PATTERN = Pattern.compile(
"allowed=.*?\\s" +
Expand All @@ -159,6 +154,9 @@ public class TestFsck {
private static final String LINE_SEPARATOR =
System.getProperty("line.separator");

private static final LogCapturer AUDIT_LOG_CAPTURE =
LogCapturer.captureLogs(FSNamesystem.AUDIT_LOG);

public static String runFsck(Configuration conf, int expectedErrCode,
boolean checkErrorCode, String... path)
throws Exception {
Expand Down Expand Up @@ -191,11 +189,6 @@ public void tearDown() throws Exception {
shutdownCluster();
}

@AfterClass
public static void afterClass() throws Exception {
assertTrue(AUDIT_LOG_FILE.delete());
}

private void shutdownCluster() throws Exception {
if (cluster != null) {
cluster.shutdown();
Expand Down Expand Up @@ -245,29 +238,30 @@ public void testFsck() throws Exception {
util.cleanup(fs, "/srcdat");
}

private void verifyAuditLogs() throws IOException {
try (BufferedReader reader = new BufferedReader(new FileReader(AUDIT_LOG_FILE))) {
// Audit log should contain one getfileinfo and one fsck
String line;
int getFileStatusSuccess = 0;
int fsckCount = 0;
while ((line = reader.readLine()) != null) {
LOG.info("Line: {}", line);
if (line.contains("cmd=getfileinfo") && GET_FILE_INFO_PATTERN.matcher(line).matches()) {
getFileStatusSuccess++;
} else if (FSCK_PATTERN.matcher(line).matches()) {
fsckCount++;
}
}
if (getFileStatusSuccess < 2) {
throw new AssertionError(
"getfileinfo cmd should occur at least 2 times. Actual count: " + getFileStatusSuccess);
private void verifyAuditLogs() {
String[] auditLogOutputLines = AUDIT_LOG_CAPTURE.getOutput().split("\\n");
int fileStatusSuccess = 0;
int fsckCount = 0;
for (String auditLogLine : auditLogOutputLines) {
if (!auditLogLine.contains("allowed=")) {
continue;
}
if (fsckCount < 1) {
throw new AssertionError(
"fsck should be present at least once. Actual count: " + fsckCount);
String extractedAuditLog = "allowed=" + auditLogLine.split("allowed=")[1];
LOG.info("Line: {}", extractedAuditLog);
if (extractedAuditLog.contains("cmd=getfileinfo") && GET_FILE_INFO_PATTERN.matcher(
extractedAuditLog).matches()) {
fileStatusSuccess++;
} else if (FSCK_PATTERN.matcher(extractedAuditLog).matches()) {
fsckCount++;
}
}
if (fileStatusSuccess < 2) {
throw new AssertionError(
"getfileinfo cmd should occur at least 2 times. Actual count: " + fileStatusSuccess);
}
if (fsckCount < 1) {
throw new AssertionError("fsck should be present at least once. Actual count: " + fsckCount);
}
}

@Test
Expand Down