Skip to content

Commit

Permalink
Avoid publishing experiment config when W&B publication is disabled (#…
Browse files Browse the repository at this point in the history
…627)

* Avoid publishing experiment config when W&B publication is disabled

* TRASHME Reproduce issue with train-backwards

* Revert "TRASHME Reproduce issue with train-backwards"

This reverts commit ad2b7a7.
  • Loading branch information
vrigal authored May 23, 2024
1 parent c2a6e7f commit c6c10a5
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions tracking/translations_parser/cli/taskcluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@ def get_args() -> argparse.Namespace:
)
parser.add_argument(
"--publish-group-logs",
help=("Enable publishing a group_logs fake run with the experiment configuration."),
help=(
"Enable publishing a group_logs fake run with the experiment configuration."
"This option requires W&B publication to be enabled, otherwise it will be ignored."
),
action="store_true",
default=False,
)
Expand Down Expand Up @@ -105,12 +108,13 @@ def boot() -> None:
)
if wandb_publisher:
publishers.append(wandb_publisher)
elif args.publish_group_logs:
logger.warning(
"Ignoring --publish-group-logs option as Weight & Biases publication is disabled."
)

# Use log filtering when using non-stream (for uploading past experiments)
log_filter = taskcluster_log_filter if not args.from_stream else None

# publish the config fist
if args.publish_group_logs:
# Publish experiment configuration before parsing the training logs
if wandb_publisher and args.publish_group_logs:
logger.info("Publishing experiment config to a 'group_logs' fake run.")
# Retrieve experiment configuration from the task group
task_id = os.environ.get("TASK_ID")
Expand All @@ -124,6 +128,8 @@ def boot() -> None:
config = task_group.get("extra", {}).get("action", {}).get("context", {}).get("input")
publish_group_logs_from_tasks(config=config)

# Use log filtering when using non-stream (for uploading past experiments)
log_filter = taskcluster_log_filter if not args.from_stream else None
parser = TrainingParser(
lines,
publishers=publishers,
Expand Down

0 comments on commit c6c10a5

Please sign in to comment.