Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 48 additions & 3 deletions src/libkernelbot/leaderboard_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -1257,7 +1257,10 @@ def get_user_submissions(
offset: Offset for pagination

Returns:
List of submission dictionaries with summary info and runs
List of submission dictionaries with summary info and runs. Each
entry includes ``status`` ("pending"/"failed"/"done"),
``public_score`` and ``secret_score`` (the geomean leaderboard
scores, either may be ``None``), plus the public ``runs`` list.
"""
# Validate and clamp inputs
limit = max(1, min(limit, 100))
Expand Down Expand Up @@ -1325,17 +1328,59 @@ def get_user_submissions(
"score": run_row[2],
})

# Per-submission status and leaderboard scores. The public `runs`
# above are ranking-filtered (anti-cheat: the public score is hidden
# unless the matching secret run passed). Here we additionally
# surface the secret leaderboard score (visible to the owner, as the
# detail endpoint already does) and whether any run failed, so
# callers can show an accurate status and both scores without an
# extra request per submission.
agg_query = """
SELECT submission_id,
MIN(score) FILTER (
WHERE mode = 'leaderboard' AND secret AND passed
) AS secret_score,
Comment on lines +1349 to +1351

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand this here. Can you explain this to me? Why MIN(score)?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A submission can have a secret leaderboard run per GPU type, so there can be more than one secret score. MIN takes the best (lowest = fastest) one, matching how the existing public score is summarized for the row (the runs are likewise per-GPU and the caller/CLI takes the min). For single-GPU leaderboards like qr_v2 there's exactly one, so MIN is just that value. Happy to switch to per-GPU secret scores instead if you'd prefer symmetry with runs, but a single ranking number seemed more useful for the list view.

bool_or(NOT passed) AS has_failed_run
FROM leaderboard.runs
WHERE submission_id = ANY(%s)
GROUP BY submission_id
Comment on lines +1348 to +1355

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you benchmark how long this sequel query will take?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. On the docker-compose test Postgres, seeded with 100 submissions × 6 runs (600 rows), the added aggregate query runs at ~0.25 ms/call for a full 100-submission page, versus ~21 ms for the whole get_user_submissions call — about 1% overhead. It's the same access pattern as the existing runs query in this method (WHERE submission_id = ANY(%s), grouped) over the same rows, so no new join or table. And it replaces what a client otherwise needs up to 100 detail round-trips to compute.

"""
self.cursor.execute(agg_query, (submission_ids,))
agg_by_submission: dict = {
row[0]: {"secret_score": row[1], "has_failed_run": row[2]}
for row in self.cursor.fetchall()
}

# Build result with runs grouped by submission
results = []
for row in submissions:
sub_id = row[0]
done = row[4]
public_runs = runs_by_submission.get(sub_id, [])
agg = agg_by_submission.get(sub_id, {})

# The public leaderboard score (lowest across GPUs), already
# ranking-eligible by construction of `runs_query`.
public_scores = [r["score"] for r in public_runs if r["score"] is not None]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm confused. I thought the public scores were already reported by the submissions endpoint. So why do we need new code to report it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right — the public score was already exposed (in runs[].score), so the separate public_score field was redundant. I've dropped it in 08d2604. The list payload now only adds the two things that genuinely weren't derivable from the existing runs: status and secret_score (the secret runs are never selected by the list query, so the secret score and the per-run passed flags don't leave the server today).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol bro come on I expected better.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that one's on me — I added a field that already existed in runs[].score without checking. Dropped it; the patch now adds only status and secret_score.

public_score = min(public_scores) if public_scores else None

if not done:
status = "pending"
elif agg.get("has_failed_run"):
status = "failed"
else:
status = "done"

results.append({
"id": sub_id,
"leaderboard_name": row[1],
"file_name": row[2],
"submission_time": row[3],
"done": row[4],
"runs": runs_by_submission.get(sub_id, []),
"done": done,
"status": status,
"public_score": public_score,
"secret_score": agg.get("secret_score"),
"runs": public_runs,
})
return results
except psycopg2.Error as e:
Expand Down
60 changes: 60 additions & 0 deletions tests/test_leaderboard_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,66 @@ def test_get_user_submissions_with_multiple_runs(database, submit_leaderboard):
assert 2.0 in scores


def test_get_user_submissions_status_and_scores_on_success(database, submit_leaderboard):
"""A fully-passing submission reports status 'done' with public+secret scores."""
with database as db:
sub = db.create_submission(
"submit-leaderboard", "ok.py", 5, "code",
datetime.datetime.now(tz=datetime.timezone.utc), user_name="user5",
)
_create_submission_run(db, sub, mode="leaderboard", secret=False, runner="A100", score=1.5)
_create_submission_run(db, sub, mode="leaderboard", secret=True, runner="A100", score=1.7)
db.mark_submission_done(sub)

result = db.get_user_submissions(user_id="5")
assert len(result) == 1
assert result[0]["status"] == "done"
# Scores come back as Decimal from Postgres (as the existing `runs`
# score does); compare as float.
assert float(result[0]["public_score"]) == 1.5
assert float(result[0]["secret_score"]) == 1.7


def test_get_user_submissions_status_failed_when_run_failed(database, submit_leaderboard):
"""A submission with a failed run reports status 'failed'."""
failed = dataclasses.replace(sample_run_result(), passed=False)
with database as db:
sub = db.create_submission(
"submit-leaderboard", "bad.py", 5, "code",
datetime.datetime.now(tz=datetime.timezone.utc), user_name="user5",
)
_create_submission_run(db, sub, mode="leaderboard", secret=False, runner="A100", score=1.5)
_create_submission_run(
db, sub, mode="leaderboard", secret=True, runner="A100",
score=None, result=failed,
)
db.mark_submission_done(sub)

result = db.get_user_submissions(user_id="5")
assert len(result) == 1
assert result[0]["status"] == "failed"
# Ranking-eligibility (and thus the public score / runs) is withheld
# when the secret run failed, but the secret score stays None too.
assert result[0]["public_score"] is None
assert result[0]["secret_score"] is None
assert result[0]["runs"] == []


def test_get_user_submissions_status_pending_when_not_done(database, submit_leaderboard):
"""A not-yet-finished submission reports status 'pending'."""
with database as db:
sub = db.create_submission(
"submit-leaderboard", "wip.py", 5, "code",
datetime.datetime.now(tz=datetime.timezone.utc), user_name="user5",
)
_create_submission_run(db, sub, mode="leaderboard", secret=False, runner="A100", score=1.5)
# Not marked done.

result = db.get_user_submissions(user_id="5")
assert len(result) == 1
assert result[0]["status"] == "pending"


def test_check_leaderboard_access_public(database, submit_leaderboard):
"""Public leaderboards grant access to everyone."""
with database as db:
Expand Down