Merge pull request #701 from allenai/dh-add-0shots

add 0 shot requests dumps for olmes

olmo_data/oe_eval_tasks/arc_easy_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "arc_easy", "task_hash": "7710384c230e8788b53fd2dbb457c074", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "native_id_field": "id", "primary_metric": "acc_per_char", "split": "validation", "fewshot_source": "OLMES:ARC-Easy", "context_kwargs": {"description": null}, "random_subsample_seed": 1234, "num_shots": 0, "limit": 1000, "metadata": {"description": "ARC-Easy (RC) using OLMES-v0.1", "regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "task_name": "arc_easy", "version": 0, "task_core": "arc_easy"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 3.5188589096069336, "current_date": "2024-08-13 19:40:32 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/boolq_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "boolq", "task_hash": "6b8adf45a10aa2c14c9560b19d502b51", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "super_glue", "dataset_name": "boolq", "native_id_field": "idx", "primary_metric": "acc_raw", "fewshot_source": "OLMES:BoolQ", "metric_kwargs": {"uncond_docid_offset": null}, "random_subsample_seed": 1234, "split": "validation", "limit": 1000, "num_shots": 0, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "fewshot_seed": 1234, "task_name": "boolq", "version": 0, "task_core": "boolq"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 2.34824800491333, "current_date": "2024-08-13 19:40:44 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/csqa_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "csqa", "task_hash": "26f9f7f97ee7ccb003ba0f436f0298a5", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "commonsense_qa", "native_id_field": "id", "primary_metric": "acc_uncond", "split": "validation", "fewshot_source": "OLMES:commonsense_qa", "random_subsample_seed": 1234, "num_shots": 0, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "dataset_name": null, "task_name": "csqa", "version": 0, "task_core": "csqa"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 6.059943199157715, "current_date": "2024-08-13 19:41:06 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/hellaswag_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "hellaswag", "task_hash": "fbd0c628d364f50a5b483584e20c54df", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "hellaswag", "native_id_field": "ind", "primary_metric": "acc_per_char", "split": "validation", "fewshot_source": "OLMES:hellaswag", "random_subsample_seed": 1234, "num_shots": 0, "limit": 1000, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "dataset_name": null, "task_name": "hellaswag", "version": 0, "task_core": "hellaswag"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 3.5681426525115967, "current_date": "2024-08-13 19:41:18 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/openbookqa_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "openbookqa", "task_hash": "42add59f3d0e066899ce1fdebe2f2a10", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "openbookqa", "dataset_name": "main", "native_id_field": "id", "primary_metric": "acc_uncond", "split": "validation", "fewshot_source": "OLMES:openbookqa", "context_kwargs": {"no_prefix": false}, "random_subsample_seed": 1234, "num_shots": 0, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "task_name": "openbookqa", "version": 0, "task_core": "openbookqa"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 6.268042087554932, "current_date": "2024-08-13 19:41:36 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/piqa_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "piqa", "task_hash": "90a18d71a03c8b369f2cf619cd796f06", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "piqa", "native_id_field": "index", "primary_metric": "acc_per_char", "split": "validation", "fewshot_source": "OLMES:piqa", "random_subsample_seed": 1234, "num_shots": 0, "limit": 1000, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "dataset_name": null, "task_name": "piqa", "version": 0, "task_core": "piqa"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 5.096452236175537, "current_date": "2024-08-13 19:43:35 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/socialiqa_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "socialiqa", "task_hash": "4b754270ac8645bc8fd4360ddc1c14ad", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "social_i_qa", "native_id_field": "index", "primary_metric": "acc_per_char", "split": "validation", "fewshot_source": "OLMES:social_i_qa", "random_subsample_seed": 1234, "num_shots": 0, "limit": 1000, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "metric_kwargs": {"uncond_docid_offset": 1000000}, "fewshot_seed": 1234, "dataset_name": null, "task_name": "socialiqa", "version": 0, "task_core": "socialiqa"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 5.03247594833374, "current_date": "2024-08-13 19:42:43 UTC", "num_instances": 0, "beaker_info": {}}

olmo_data/oe_eval_tasks/winogrande_rc_0shot/config.json

@@ -0,0 +1 @@
+{"task_name": "winogrande", "task_hash": "efea61bbeb822d4e6c2fc2a971452718", "model_hash": "f72d3d62f623630ef7bb65f61a297fd6", "model_config": {"model": null, "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"dataset_path": "winogrande", "dataset_name": "winogrande_xl", "native_id_field": null, "primary_metric": "acc_raw", "split": "validation", "fewshot_source": "OLMES:winogrande", "random_subsample_seed": 1234, "num_shots": 0, "metadata": {"regimes": ["OLMES-v0.1"]}, "generation_kwargs": {}, "context_kwargs": {}, "metric_kwargs": {}, "fewshot_seed": 1234, "task_name": "winogrande", "version": 0, "task_core": "winogrande"}, "compute_config": {"batch_size": "4", "max_batch_size": 32, "output_dir": "my_requests", "num_recorded_inputs": 3, "save_raw_requests": true, "check_datalake": false}, "processing_time": 3.8086090087890625, "current_date": "2024-08-13 19:43:58 UTC", "num_instances": 0, "beaker_info": {}}