Skip to content
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions configs/inference/formatask.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
dataset = "kalomaze/general-formatask-it1-7k"
batch_size = 256
dp = 4
rollout_path = "outputs"
Comment thread
kalomaze marked this conversation as resolved.
Resolved
output_path = "data_rollout"
max_model_len = 2048

[sampling]
temperature = 1.0
n = 16
23 changes: 23 additions & 0 deletions configs/training/formatask.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
project = "formatask-debug"

[train]
micro_bs = 2 # change to 8 for H200
reshard_after_forward = true

[optim]
batch_size = 64
warmup_steps = 20
total_steps = 100000000000000
step_per_rollout = 4
grad_norm_clip = 0.0001

[optim.optim]
lr = 5e-6

[data]
path = "data_rollout"
seq_length = 2048

[ckpt]
rollout_path = "outputs"
Comment thread
kalomaze marked this conversation as resolved.
Resolved
3 changes: 3 additions & 0 deletions src/zeroband/inference/genesys/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from zeroband.inference.genesys.reasoning_gym import verify_reasoning_gym
from zeroband.inference.genesys.reverse_text import reverse_text
from zeroband.inference.genesys.unscramble_sentence import compute_reward as compute_unscramble_reward
from zeroband.inference.genesys.formatask import compute_reward as compute_formatask_reward

TaskType = Literal[
"verifiable_math",
Expand All @@ -20,6 +21,7 @@
"ascii_tree_formatting",
"pydantic_adherance",
"complex_json_output",
"formatask",
]


Expand All @@ -40,4 +42,5 @@ def get_reward_function(task_type: TaskType) -> Callable[[str, dict], float]:
"ascii_tree_formatting": compute_ascii_tree_reward,
"pydantic_adherance": validate_pydantic_json,
"complex_json_output": verify_complex_json_formatting,
"formatask": compute_formatask_reward,
}
50 changes: 50 additions & 0 deletions src/zeroband/inference/genesys/formatask.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import difflib
import re
from typing import Dict


def compute_reward(
completion: str,
verification_info: Dict,
tag_name: str = "extracted_formatted"
):
"""
Generic difflib-based reward computation for tasks expecting extracted content in XML tags.

Args:
completion: The model's completion text
verification_info: Dictionary containing ground truth
tag_name: XML tag name to extract content from

Returns:
Float reward between 0 and 1
"""
# Extract answer from specified tag
tag_pattern = f"<{tag_name}>(.*?)</{tag_name}>"
if f"<{tag_name}>" not in completion:
return 0

answer_match = re.search(tag_pattern, completion, re.DOTALL)
if not answer_match:
return 0

# Get ground truth
ground_truth = verification_info.get("ground_truth")
if not ground_truth:
return 0

try:
# Clean and split both into lines
answer_lines = answer_match.group(1).strip().split("\n")
truth_lines = ground_truth.strip().split("\n")

# Use difflib to compare line sequences
matcher = difflib.SequenceMatcher(None, answer_lines, truth_lines)

# Calculate similarity ratio
reward = matcher.ratio()

return reward

except Exception:
return 0