Skip to content

HDFS-17528. FsImageValidation: set txid when saving a new image #6828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szetszwo merged 9 commits into from
Jun 19, 2024
+14 −6
Merged

HDFS-17528. FsImageValidation: set txid when saving a new image #6828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5c39104
HDFS-17528. FsImageValidation: set txid when saving a new image
szetszwo May 14, 2024
ff73381
empty commit to trigger tests
szetszwo May 20, 2024
3fcd4c9
empty commit to trigger tests 2
szetszwo May 20, 2024
1d3dc13
Empty commit to trigger test 3
szetszwo May 30, 2024
6f9f956
One last time.
szetszwo Jun 3, 2024
256d41d
Resubmit
szetszwo Jun 11, 2024
d61a831
Retest
szetszwo Jun 11, 2024
972adb7
Retest again.
szetszwo Jun 12, 2024
173dab6
Retest
szetszwo Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions ...dfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -977,12 +977,16 @@ private void loadFSImage(File curFile, MD5Hash expectedMd5,
" but expecting " + expectedMd5);
}

long txId = loader.getLoadedImageTxId();
final long txId = setLastAppliedTxId(loader);
LOG.info("Loaded image for txid " + txId + " from " + curFile);
lastAppliedTxId = txId;
storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}

synchronized long setLastAppliedTxId(FSImageFormat.LoaderDelegator loader) {
lastAppliedTxId = loader.getLoadedImageTxId();
return lastAppliedTxId;
}

/**
* Save the contents of the FS image to the file.
*/
Expand Down Expand Up @@ -1215,8 +1219,9 @@ public void removeFromCheckpointing(long txid) {
}

void save(FSNamesystem src, File dst) throws IOException {
final SaveNamespaceContext context = new SaveNamespaceContext(src,
getCorrectLastAppliedOrWrittenTxId(), new Canceler());
final long txid = getCorrectLastAppliedOrWrittenTxId();
LOG.info("save fsimage with txid={} to {}", txid, dst.getAbsolutePath());
final SaveNamespaceContext context = new SaveNamespaceContext(src, txid, new Canceler());
final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst);
Files.createDirectories(storageDirectory.getCurrentDir().toPath());
new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run();
Expand Down
7 changes: 5 additions & 2 deletions ...t/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception {
initConf(conf);

// check INodeReference
NameNode.initMetrics(conf, HdfsServerConstants.NamenodeRole.NAMENODE); // to avoid NPE
final FSNamesystem namesystem = checkINodeReference(conf, errorCount);

// check INodeMap
Expand Down Expand Up @@ -276,14 +277,16 @@ public void run() {
namesystem.getFSDirectory().writeLock();
try {
loader.load(fsImageFile, false);
fsImage.setLastAppliedTxId(loader);
} finally {
namesystem.getFSDirectory().writeUnlock();
namesystem.writeUnlock("loadImage");
}
}
t.cancel();
Cli.println("Loaded %s %s successfully in %s",
FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart));
Cli.println("Loaded %s %s with txid %d successfully in %s",
FS_IMAGE, fsImageFile, namesystem.getFSImage().getLastAppliedTxId(),
StringUtils.formatTime(now() - loadStart));
return namesystem;
}

Expand Down