Eval hackathon

promptsource/templates/wmt14/cs-en/templates.yaml

@@ -0,0 +1,345 @@
+dataset: wmt14
+subset: cs-en
+templates:
+  027bdc9b-aed7-4509-94ea-38b138f14ddf: !Template
+    answer_choices: null
+    id: 027bdc9b-aed7-4509-94ea-38b138f14ddf
+    jinja: 'Translate this from Czech into English: {{translation["cs"]}}
+
+      |||  {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate-cs-en-source+target
+    reference: https://arxiv.org/pdf/1910.10683.pdf
+  16da9664-30e5-4acc-aff6-21658158cf85: !Template
+    answer_choices: null
+    id: 16da9664-30e5-4acc-aff6-21658158cf85
+    jinja: 'What is the Czech translation of: {{translation["en"]}}
+
+      |||  {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: gpt-3-en-cs-target
+    reference: GPT-3 paper
+  3b3ca652-8414-4442-b55f-c2fc7630b071: !Template
+    answer_choices: null
+    id: 3b3ca652-8414-4442-b55f-c2fc7630b071
+    jinja: 'Given the following source text in English: {{translation["en"]}} , a
+      good Czech translation is:
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: a_good_translation-en-cs-source+target
+    reference: ''
+  4266e99a-41fa-4c2d-8bbc-91317f42d9ff: !Template
+    answer_choices: null
+    id: 4266e99a-41fa-4c2d-8bbc-91317f42d9ff
+    jinja: 'What is the English translation of the Czech  sentence: {{translation["cs"]}}
+
+      |||  {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: gpt-3-cs-en-source+target
+    reference: GPT-3 paper
+  46621847-e018-43fe-b8d1-af439e130254: !Template
+    answer_choices: null
+    id: 46621847-e018-43fe-b8d1-af439e130254
+    jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}}
+
+      |||  {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: gpt-3-en-cs-source+target
+    reference: GPT 3 paper
+  4c52dc6a-f16c-4f3c-99d0-274b4162a616: !Template
+    answer_choices: null
+    id: 4c52dc6a-f16c-4f3c-99d0-274b4162a616
+    jinja: 'If the original version says: {{translation["en"]}}; then the Czech version
+      should say:
+
+      |||   {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: version-en-cs-target
+    reference: ''
+  5b87693b-ed41-4794-bb3e-b9bf7ada2689: !Template
+    answer_choices: null
+    id: 5b87693b-ed41-4794-bb3e-b9bf7ada2689
+    jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation
+      is: ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: a_good_translation-en-cs-target
+    reference: ''
+  5e476fb7-5e6e-4732-a084-88e361a82de0: !Template
+    answer_choices: null
+    id: 5e476fb7-5e6e-4732-a084-88e361a82de0
+    jinja: '{{translation["cs"]}}  translates into English as:
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate_as_cs-en-target
+    reference: ''
+  744bab4b-44d1-4368-a98d-4a403c1a7e0e: !Template
+    answer_choices: null
+    id: 744bab4b-44d1-4368-a98d-4a403c1a7e0e
+    jinja: 'Translate this into English: {{translation["cs"]}}
+
+      |||  {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate-cs-en-target
+    reference: ''
+  7fd9d8d7-0c72-42da-a029-ef4d9a9e5f72: !Template
+    answer_choices: null
+    id: 7fd9d8d7-0c72-42da-a029-ef4d9a9e5f72
+    jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good
+      English translation is: ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: a_good_translation-cs-en-source+target
+    reference: ''
+  809cc7a3-189b-4018-8f81-1e65c2f42271: !Template
+    answer_choices: null
+    id: 809cc7a3-189b-4018-8f81-1e65c2f42271
+    jinja: 'Given the following passage:  {{translation["cs"]}} , a good English translation
+      is:
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: a_good_translation-cs-en-target
+    reference: ''
+  9023b083-ccf1-42a4-9589-5fffd3562649: !Template
+    answer_choices: null
+    id: 9023b083-ccf1-42a4-9589-5fffd3562649
+    jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version
+      should say:
+
+      |||   {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: version-cs-en-source+target
+    reference: ''
+  92e3a4a7-e7c2-4ea1-abfb-be63069d13fa: !Template
+    answer_choices: null
+    id: 92e3a4a7-e7c2-4ea1-abfb-be63069d13fa
+    jinja: 'Translate this into Czech: {{translation["en"]}}
+
+      |||  {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate-en-cs-target
+    reference: ''
+  a63daae0-77ba-4790-a89d-2428193fbcdc: !Template
+    answer_choices: null
+    id: a63daae0-77ba-4790-a89d-2428193fbcdc
+    jinja: 'What is the English translation of : {{translation["cs"]}}
+
+      |||  {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: gpt-3-cs-en-target
+    reference: GPT-3 paper
+  a63fd691-4f07-41da-a98c-a934e866e74e: !Template
+    answer_choices: null
+    id: a63fd691-4f07-41da-a98c-a934e866e74e
+    jinja: '{{translation["cs"]}} = English:
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: xglm-cs-en-target
+    reference: XGLM paper https://arxiv.org/abs/2112.10668
+  ab264706-636e-4bd0-a762-860029b54f45: !Template
+    answer_choices: null
+    id: ab264706-636e-4bd0-a762-860029b54f45
+    jinja: 'Translate this from English into Czech: {{translation["en"]}}
+
+      |||  {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate-en-cs-source+starget
+    reference: ''
+  ac25c74b-1a81-409e-9c4f-bdc94912721c: !Template
+    answer_choices: null
+    id: ac25c74b-1a81-409e-9c4f-bdc94912721c
+    jinja: ' {{translation["en"]}}  translates into Czech as:
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate_as_en-cs-target
+    reference: ''
+  b42858a8-35d7-4172-ace4-80df1f6a94e0: !Template
+    answer_choices: null
+    id: b42858a8-35d7-4172-ace4-80df1f6a94e0
+    jinja: 'If the English version says: {{translation["en"]}}; then the Czech version
+      should say:
+
+      |||   {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: version-en-cs-source+target
+    reference: ''
+  c4629789-74ce-4378-8d03-f75b2327ad77: !Template
+    answer_choices: null
+    id: c4629789-74ce-4378-8d03-f75b2327ad77
+    jinja: 'English: {{translation["en"]}} = Czech:
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: xglm-en-cs-source-target
+    reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668
+  cdce6dd9-52ad-42c3-a691-8efb13ab535f: !Template
+    answer_choices: null
+    id: cdce6dd9-52ad-42c3-a691-8efb13ab535f
+    jinja: 'Czech:  {{translation["cs"]}}  translates into English as:
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate_as_cs-en-source+target
+    reference: ''
+  dcefc4dc-560d-4bf5-8623-c99d535232b0: !Template
+    answer_choices: null
+    id: dcefc4dc-560d-4bf5-8623-c99d535232b0
+    jinja: 'How do you say {{translation["cs"]}} in English?
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: how_to_say-cs-en-target
+    reference: ''
+  e13ee973-fce4-4d42-8ec5-b871c20eaf39: !Template
+    answer_choices: null
+    id: e13ee973-fce4-4d42-8ec5-b871c20eaf39
+    jinja: 'Czech: {{translation["cs"]}} = English:
+
+      ||| {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: xglm-cs-en-source+target
+    reference: XGLM paper https://arxiv.org/abs/2112.10668
+  ee298b4c-8165-44cc-81b6-1fb18f4e9e02: !Template
+    answer_choices: null
+    id: ee298b4c-8165-44cc-81b6-1fb18f4e9e02
+    jinja: 'English:  {{translation["en"]}}  translates into Czech as:
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: translate_as_en-cs-source+target
+    reference: ''
+  f7af3145-3444-4183-921b-a549fef5e1b3: !Template
+    answer_choices: null
+    id: f7af3145-3444-4183-921b-a549fef5e1b3
+    jinja: 'If the original version says: {{translation["cs"]}}; then the English
+      version should say:
+
+      |||   {{translation["en"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: version-cs-en-target
+    reference: ''
+  f7d630ca-1791-4aaf-97b5-7c15e5eba296: !Template
+    answer_choices: null
+    id: f7d630ca-1791-4aaf-97b5-7c15e5eba296
+    jinja: 'How do you say {{translation["en"]}} in Czech?
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: how_to_say-en-cs-target
+    reference: ''
+  f83cb5c3-7989-49d9-a1cb-6fdc16b445e3: !Template
+    answer_choices: null
+    id: f83cb5c3-7989-49d9-a1cb-6fdc16b445e3
+    jinja: '{{translation["en"]}} = Czech:
+
+      ||| {{translation["cs"]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: xglm-en-cs-target
+    reference: XGLM paper https://arxiv.org/abs/2112.10668