v1.0 - Initial stable version

• Supports C++, Python, Java, JavaScript (Node.js), C#.
• Verdicts: Solved, Wrong answer, Time limit exceeded, Memory limit exceeded, Output limit exceeded, Runtime error, Compilation error, Skipped.
• Supports files (input_files and target_files)
• Support for projects (multi-file submissions)
• Linting through pre-commit
• Test coverage

Evaluate many submissions:

import random
import time
from pathlib import Path

import boto3
import json
import glob
from multiprocessing import Pool

from models import SubmissionRequest, SubmissionResult, RunResult, Status


def evaluate(submission: SubmissionRequest) -> SubmissionResult:
    try:
        lambdas = boto3.client('lambda')
        response = lambdas.invoke(
            FunctionName='Bouncer',
            InvocationType='RequestResponse',
            Payload=json.dumps({'body': submission.to_json()}),
        )

        res = response['Payload'].read().decode('utf-8')
        res = json.loads(res)
        # print('res:', res)
        return SubmissionResult.from_json(res['body'])
    except:
        print('Something went wrong:', submission.problem)
        return SubmissionResult(overall=RunResult(status=Status.WA, memory=0, time=0, return_code=0),
                                compile_result=RunResult(status=Status.WA, memory=0, time=0, return_code=0))


def evaluate_all():
    all_submissions = glob.glob('../submissions/*.cpp') + glob.glob('../submissions/*.py')

    # all_submissions = all_submissions[:10]
    # all_submissions = ['../submissions/000787-D.cpp']
    random.shuffle(all_submissions)
    print(all_submissions)
    print(len(all_submissions))
    submissions = [SubmissionRequest(
        language='python' if s.endswith('.py') else 'C++11',
        code={'main.py' if s.endswith('.py') else 'main.cpp': Path(s).read_text()},
        problem=s.split('-')[-1].split('.')[0],     # 001122-J.cpp => J
    ) for s in all_submissions]

    with Pool(100) as p:
        results = p.starmap(evaluate, [(submission,) for submission in submissions])

    final = {submission.split('/')[-1].split('-')[0]: 'OK' if res.overall.status == Status.OK else 'FAIL'
             for submission, res in zip(all_submissions, results)}
    return final


if __name__ == '__main__':
    start_time = time.time()
    all_results = evaluate_all()
    print('--------------------')
    with open('results-new.txt', 'w') as f:
        for k, v in sorted(all_results.items(), key=lambda kv: kv[0]):
            print(f'{k}: {v}')
            f.write(f'{k}: {v}\n')

    print('Took:', time.time() - start_time, 'seconds')