Skip to content

HDDS-10190. [hsync] Handle lease recovery for file without blocks. #6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jojochuang merged 2 commits into from
Jan 26, 2024
Merged

HDDS-10190. [hsync] Handle lease recovery for file without blocks. #6094

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2fbf3e0
HDDS-10190. Handle lease recovery for file without blocks.
ashishkr200 Jan 25, 2024
e576685
Fix review comment
ashishkr200 Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ...op-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,27 @@ public void testRecoverWrongFile() throws Exception {
}
}

@Test
public void testRecoveryWithoutBlocks() throws Exception {
RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf);

final FSDataOutputStream stream = fs.create(file, true);
try {
stream.hsync();
assertFalse(fs.isFileClosed(file));

int count = 0;
while (count++ < 15 && !fs.recoverLease(file)) {
Thread.sleep(1000);
}
// The lease should have been recovered.
assertTrue(fs.isFileClosed(file), "File should be closed");

} finally {
closeIgnoringKeyNotFound(stream);
}
}

private void verifyData(byte[] data, int dataSize, Path filePath, RootedOzoneFileSystem fs) throws IOException {
try (FSDataInputStream fdis = fs.open(filePath)) {
int bufferSize = dataSize > data.length ? dataSize / 2 : dataSize;
Expand Down
32 changes: 17 additions & 15 deletions ...-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,29 +248,31 @@ private RecoverLeaseResponse doWork(OzoneManager ozoneManager,
List<OmKeyLocationInfo> keyLocationInfoList = keyLatestVersionLocations.getLocationList();
OmKeyLocationInfoGroup openKeyLatestVersionLocations = openKeyInfo.getLatestVersionLocations();
List<OmKeyLocationInfo> openKeyLocationInfoList = openKeyLatestVersionLocations.getLocationList();
OmKeyLocationInfo finalBlock;

OmKeyLocationInfo finalBlock = null;
boolean returnKeyInfo = true;
if (openKeyLocationInfoList.size() > keyLocationInfoList.size() &&
openKeyModificationTime > keyInfo.getModificationTime()) {
openKeyModificationTime > keyInfo.getModificationTime() &&
openKeyLocationInfoList.size() > 0) {
finalBlock = openKeyLocationInfoList.get(openKeyLocationInfoList.size() - 1);
returnKeyInfo = false;
} else {
} else if (keyLocationInfoList.size() > 0) {
finalBlock = keyLocationInfoList.get(keyLocationInfoList.size() - 1);
}

// set token to last block if enabled
if (ozoneManager.isGrpcBlockTokenEnabled()) {
String remoteUser = getRemoteUser().getShortUserName();
OzoneBlockTokenSecretManager secretManager = ozoneManager.getBlockTokenSecretManager();
finalBlock.setToken(secretManager.generateToken(remoteUser, finalBlock.getBlockID(),
EnumSet.of(READ, WRITE), finalBlock.getLength()));
if (finalBlock != null) {
// set token to last block if enabled
if (ozoneManager.isGrpcBlockTokenEnabled()) {
String remoteUser = getRemoteUser().getShortUserName();
OzoneBlockTokenSecretManager secretManager = ozoneManager.getBlockTokenSecretManager();
finalBlock.setToken(secretManager.generateToken(remoteUser, finalBlock.getBlockID(),
EnumSet.of(READ, WRITE), finalBlock.getLength()));
}
// refresh last block pipeline
ContainerWithPipeline containerWithPipeline =
ozoneManager.getScmClient().getContainerClient().getContainerWithPipeline(finalBlock.getContainerID());
finalBlock.setPipeline(containerWithPipeline.getPipeline());
}

// refresh last block pipeline
ContainerWithPipeline containerWithPipeline =
ozoneManager.getScmClient().getContainerClient().getContainerWithPipeline(finalBlock.getContainerID());
finalBlock.setPipeline(containerWithPipeline.getPipeline());

RecoverLeaseResponse.Builder rb = RecoverLeaseResponse.newBuilder();
rb.setKeyInfo(returnKeyInfo ? keyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true) :
openKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(), true));
Expand Down
18 changes: 10 additions & 8 deletions hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,17 @@ public boolean recoverLease(Path f) throws IOException {

// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList = keyInfo.getLatestVersionLocations().getLocationList();
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
if (!locationInfoList.isEmpty()) {
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}

// recover and commit file
Expand Down
18 changes: 10 additions & 8 deletions ...ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,15 +159,17 @@ public boolean recoverLease(final Path f) throws IOException {

// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList = keyInfo.getLatestVersionLocations().getLocationList();
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
if (!locationInfoList.isEmpty()) {
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}

// recover and commit file
Expand Down
18 changes: 10 additions & 8 deletions hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,17 @@ public boolean recoverLease(Path f) throws IOException {

// finalize the final block and get block length
List<OmKeyLocationInfo> locationInfoList = keyInfo.getLatestVersionLocations().getLocationList();
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
if (!locationInfoList.isEmpty()) {
OmKeyLocationInfo block = locationInfoList.get(locationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}

// recover and commit file
Expand Down
18 changes: 10 additions & 8 deletions hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,15 +152,17 @@ public boolean recoverLease(final Path f) throws IOException {

// finalize the final block and get block length
List<OmKeyLocationInfo> keyLocationInfoList = keyInfo.getLatestVersionLocations().getLocationList();
OmKeyLocationInfo block = keyLocationInfoList.get(keyLocationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
if (!keyLocationInfoList.isEmpty()) {
OmKeyLocationInfo block = keyLocationInfoList.get(keyLocationInfoList.size() - 1);
try {
block.setLength(getAdapter().finalizeBlock(block));
} catch (Throwable e) {
if (!forceRecovery) {
throw e;
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}
LOG.warn("Failed to finalize block. Continue to recover the file since {} is enabled.",
FORCE_LEASE_RECOVERY_ENV, e);
}

// recover and commit file
Expand Down