Skip to content

Commit

Permalink
exp manager update
Browse files Browse the repository at this point in the history
Signed-off-by: Paarth Neekhara <[email protected]>
  • Loading branch information
paarthneekhara committed Dec 29, 2023
1 parent 48bcb46 commit 973de2b
Showing 1 changed file with 2 additions and 1 deletion.
3 changes: 2 additions & 1 deletion nemo/utils/exp_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -601,11 +601,12 @@ def check_resume(

trainer._checkpoint_connector.resume_from_checkpoint_fit_path = str(checkpoint)

trainer.strategy.barrier()
if is_global_rank_zero():
# Check to see if any files exist that need to be moved
files_to_move = []
for child in Path(log_dir).iterdir():
if child.is_file():
if child.is_file() and not child.name.startswith("events.out.tfevents"):
files_to_move.append(child)

if len(files_to_move) > 0:
Expand Down

0 comments on commit 973de2b

Please sign in to comment.